big converter cleanup (a4bfc251) · Commits · Zolnierczuk, Piotr / pysen

Makefile

+3 −2

Original line number	Diff line number	Diff line
		@@ -13,7 +13,8 @@ endif
		PROJVER=$(shell $(PYTHON) -c "from pysen.revision import __version__; print(__version__)")
		PROJECT=$(PROJNAME)-$(PROJVER)


		# which module to run through pylint
		LINTMOD=$(PROJNAME)

		all: build

		@@ -38,7 +39,7 @@ runtest:
		@PYTHONPATH=. $(PYTHON) -m pytest ./test

		pylint:
		@PYTHONPATH=. pylint --rcfile=.pylint.rc pysen
		@PYTHONPATH=. pylint --rcfile=.pylint.rc $(LINTMOD)

		clean: clean-docs
		@-$(PYTHON) setup.py clean

pysen/inout/init.py

+1 −2

Original line number	Diff line number	Diff line
		"""
		pysen I/O module
		"""
		#from .hdf import HdfConverter, convert_to_hdf # NOQA
		from .converters import convert_to_hdf
		from .hdf import convert_to_hdf, convert_files # NOQA
		from .reader import ( read_echo, read_magnetic, read_xyz, # NOQA
		read_detimage, read_datfile, read_diffrun, # NOQA
		read_datreat , read_transmission ) # NOQA

pysen/inout/convert2hdf.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -4,7 +4,7 @@
		import argparse

		from pysen import version, setup_logger
		from pysen.inout.converters import convert_file1 as convert_file
		from pysen.inout import convert_files

		def main():
		"the main"
		@@ -26,7 +26,8 @@ def main():

		args = parser.parse_args()
		setup_logger(args.loglevel)
		convert_file(args.filename, args.out, compression=args.compression, overwrite=args.overwrite)
		convert_files(args.filename, args.out, compression=args.compression,
		overwrite=args.overwrite, converter='from_taco')

		if __name__ == "__main__":
		main()

pysen/inout/converters.py

deleted100644 → 0

+0 −100

Original line number	Diff line number	Diff line
		"""
		File type translators
		"""
		import os.path
		import logging

		from ..revision import version as version_info

		from .hdf import HdfConverter, read_nexus_echo
		from .reader import read_echo, read_xyz, read_magnetic
		from .writer import EchoWriter, DiffractionWriter, TransmissionWriter
		from .utils import get_nsefiletype, is_file_newer

		def _make_outfilename(filename, filetype=None, outdir='.'):
		""" """
		outfile, _ = os.path.splitext(filename)
		if 'nexus' in filetype:
		outfile, _ = os.path.splitext(outfile)
		outfile = os.path.join(outdir, outfile + ".h5")
		#print(f"\noutfile={outfile}")
		return outfile

		def convert_to_hdf(filename, outdir, **kwargs):
		"""convert .echo, xyz analysis .xyz or (old) magnetic .dat file into an HDF5 file

		@note:
		will not overwrite the .h5 if it exists
		and it is newer than the .echo file

		"""
		data_reader = { 'echo' : read_echo,
		'xyz' : read_xyz,
		'magnetic' : read_magnetic,
		'nexus_echo' : read_nexus_echo,
		#
		'application/nse-echo' : read_echo,
		'application/nse-xyz' : read_xyz,
		'application/nexus-nse-echo' : read_nexus_echo,
		}

		data_type = kwargs.pop('data_type', None)
		mode = kwargs.pop('mode', 'w')
		schema = kwargs.pop('schema', None)
		compression = kwargs.pop('compression', None)
		overwrite = kwargs.get('overwrite', False)

		_log = logging.getLogger()
		# outfile
		filetype = get_nsefiletype(filename)
		basename = os.path.basename(filename)
		outfile = _make_outfilename(basename, outdir=outdir, filetype=filetype)

		_log.info('converting %s to HDF5', basename)
		if not overwrite and is_file_newer(outfile, filename):
		_log.info('file %s is newer than %s', outfile, filename)
		return outfile

		if data_type is None: # guess data type based on file
		reader = data_reader.get(filetype)
		else: # data type given
		reader = data_reader.get(data_type)
		data = reader(filename)
		converter = HdfConverter(outfile, mode=mode, version=version_info())
		converter.write(data, compression=compression, schema=schema)
		return outfile

		def convert_file1(filenames, outdir, **kwargs):
		"""convert a TACO/NeXus file into an pysen/HDF file """
		log = logging.getLogger()
		for filename in filenames:
		basename = os.path.basename(filename)
		try:
		outfile = convert_to_hdf(filename, outdir, **kwargs)
		log.info('converted %s to %s', basename, outfile)
		except RuntimeError as exc:
		log.warning("error converting %s: %s", basename, exc)
		except OSError as exc:
		log.warning("error converting %s: %s", filename, exc.strerror)



		WriterClassType = { 'application/nexus-nse-echo' : EchoWriter,
		'application/nexus-nse-diffraction' : DiffractionWriter,
		'application/nexus-nse-transmission' : TransmissionWriter }
		def convert_file2(filenames, outdir, **kwargs):
		"convert file"
		log = logging.getLogger()
		for filename in filenames:
		if not os.path.exists(filename):
		log.warning("file '%s' does not exist", filename)
		continue
		file_type = get_nsefiletype(filename)
		writer_class = WriterClassType.get(file_type, None)
		if not isinstance(writer_class, type):
		log.warning("file '%s' unknown file type %s", filename, file_type)
		continue
		writer = writer_class()
		if not writer.read_nexus(filename, **kwargs):
		continue
		writer.save(outdir=outdir)

pysen/inout/hdf.py

+102 −5

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		"""Convert .echo to HDF5"""
		import logging
		"""
		File type translators

		Convert .echo to HDF5
		"""
		import os.path
		import logging
		#
		import h5py
		import numpy as np

		#
		from ..revision import version as version_info
		from ..iostrings import encode
		from .reader import read_echo
		from .writer import EchoWriter
		#
		from .reader import read_echo, read_xyz, read_magnetic
		from .writer import EchoWriter, DiffractionWriter, TransmissionWriter
		from .utils import get_nsefiletype, is_file_newer

		#FIXME: we need to work on the idea of a schema (or not???)
		SCHEMA_DEFAULT = {
		@@ -179,3 +187,92 @@ def read_nexus_echo(filename, outdir='.'):
		raise RuntimeError("Error reading NeXus file {filename}")
		echofile = echo.save(outdir)
		return read_echo(echofile)

		# CONVERTER
		def _make_outfilename(filename, filetype=None, outdir='.'):
		""" """
		outfile, _ = os.path.splitext(filename)
		if 'nexus' in filetype:
		outfile, _ = os.path.splitext(outfile)
		outfile = os.path.join(outdir, outfile + ".h5")
		#print(f"\noutfile={outfile}")
		return outfile

		def convert_to_hdf(filename, outdir, **kwargs):
		"""convert .echo, xyz analysis .xyz or (old) magnetic .dat file into an HDF5 file

		@note:
		will not overwrite the .h5 if it exists
		and it is newer than the .echo file

		"""
		data_reader = { 'echo' : read_echo,
		'xyz' : read_xyz,
		'magnetic' : read_magnetic,
		'nexus_echo' : read_nexus_echo,
		#
		'application/nse-echo' : read_echo,
		'application/nse-xyz' : read_xyz,
		'application/nexus-nse-echo' : read_nexus_echo,
		}

		data_type = kwargs.pop('data_type', None)
		mode = kwargs.pop('mode', 'w')
		schema = kwargs.pop('schema', None)
		compression = kwargs.pop('compression', None)
		overwrite = kwargs.get('overwrite', False)

		_log = logging.getLogger()
		# outfile
		filetype = get_nsefiletype(filename)
		basename = os.path.basename(filename)
		outfile = _make_outfilename(basename, outdir=outdir, filetype=filetype)

		_log.info('converting %s to HDF5', basename)
		if not overwrite and is_file_newer(outfile, filename):
		_log.info('file %s is newer than %s', outfile, filename)
		return outfile

		if data_type is None: # guess data type based on file
		reader = data_reader.get(filetype)
		else: # data type given
		reader = data_reader.get(data_type)
		data = reader(filename)
		converter = HdfConverter(outfile, mode=mode, version=version_info())
		converter.write(data, compression=compression, schema=schema)
		return outfile



		WriterClassType = { 'application/nexus-nse-echo' : EchoWriter,
		'application/nexus-nse-diffraction' : DiffractionWriter,
		'application/nexus-nse-transmission' : TransmissionWriter }

		def convert_to_taco(filename, outdir, **kwargs):
		"""convert *.nxs.h5 file into old TACO style
		echo, translation,diffrun dat file
		"""
		file_type = get_nsefiletype(filename)
		writer_class = WriterClassType.get(file_type, None)
		if not isinstance(writer_class, type):
		raise RuntimeError("file {filename} unknown file type {file_type}")
		writer = writer_class()
		if not writer.read_nexus(filename, **kwargs):
		return None
		res = writer.save(outdir=outdir)
		return res

		def convert_files(filenames, outdir, **kwargs):
		"""convert *.nxs.h5 file to/from old TACO style files"""
		log = logging.getLogger()
		converter = { 'to_taco' : convert_to_taco,
		'from_taco' : convert_to_hdf}.get(kwargs.pop('converter', 'from_taco'))
		for filename in filenames:
		try:
		basename = os.path.basename(filename)
		outfile = converter(filename, outdir, **kwargs)
		log.info('converted %s to %s', basename, outfile)
		except RuntimeError as exc:
		log.warning("error converting %s: %s", basename, exc)
		except OSError as exc:
		log.warning("error converting %s: %s", basename, exc.strerror)