refactoring file conversion (880ef877) · Commits · Zolnierczuk, Piotr / pysen

Makefile

+3 −1

Original line number	Diff line number	Diff line
		@@ -32,7 +32,7 @@ install-user: build-ui
		uninstall:
		pip uninstall $(PROJNAME)

		test: runtest
		est: runtest

		runtest:
		@PYTHONPATH=. $(PYTHON) -m pytest ./test
		@@ -66,4 +66,6 @@ build-ui:
		clean-ui:
		@$(MAKE) -C pysen/ui clean

		rebuild-ui: clean-ui build-ui

		.PHONY: all build build-dist build-ui build-docs install-user test runtest pylint clean clean-ui clean-docs distclean

pysen/inout/init.py

+3 −2

Original line number	Diff line number	Diff line
		"""
		pysen I/O module
		"""
		from .hdf import HdfConverter, convert_to_hdf # NOQA
		#from .hdf import HdfConverter, convert_to_hdf # NOQA
		from .converters import convert_to_hdf
		from .reader import ( read_echo, read_magnetic, read_xyz, # NOQA
		read_detimage, read_datfile, read_diffrun, # NOQA
		read_datreat , read_transmission ) # NOQA
		from .writer import EchoWriter, DiffractionWriter, TransmissionWriter # NOQA
		from .scans import EchoScan, DiffractionScan, TransmissionScan, XYZScan # NOQA
		from .utils import get_nsefiletype
		# from .legacy_writer import write_csv, generate_echo # NOQA
		from .legacy_writer import write_csv # NOQA
		# from .legacy_history import extract_data, list_variables, parse_date # NOQA

pysen/inout/convert2hdf.py

+5 −27

Original line number	Diff line number	Diff line
		#!/usr/bin/env python

		""" a script to convert .echo files to .hfd5 format
		"""
		import os.path
		import argparse
		import logging

		from pysen import version, setup_logger
		from pysen.inout import convert_to_hdf #, get_nsefiletype

		def convert_file(filenames, outdir, **kwargs):
		"""convert a TACO/NeXus file into an pysen/HDF file """
		log = logging.getLogger()
		for filename in filenames:
		# get the file extension and the header
		basename = os.path.basename(filename)
		#filetype = get_nsefiletype(filename)
		try:
		outfile = convert_to_hdf(filename, outdir, **kwargs)
		log.info('converted %s to %s', basename, outfile)
		except RuntimeError as exc:
		log.warning("error converting %s: %s", basename, exc)
		except OSError as exc:
		log.warning("error converting %s: %s", filename, exc.strerror)
		from pysen.inout.converters import convert_file1 as convert_file

		def main():
		"the main"
		parser = argparse.ArgumentParser(description='convert echo file into HDF5')
		parser.set_defaults(loglevel=1, compression='gzip',
		convert='echo', overwrite=False)
		parser.add_argument('filename', metavar='file', nargs='+', help='file to process')

		parser.add_argument('--outdir', '-o', dest='out', default='.',
		help="set output directory (default '%(default)s')")
		parser.set_defaults(loglevel=1, compression='gzip', out='.', convert='echo', overwrite=False)
		parser.add_argument('filename', metavar='file', nargs='+', help='filenames to process')
		parser.add_argument('--outdir', '-o', dest='out',
		help='set output directory (default %(default)s)')
		parser.add_argument('--force','-f', dest='overwrite', action='store_true',
		help='force conversion, i.e. overwrite existing HDF files')
		parser.add_argument('--no-compression', '-C', dest='compression', action='store_const',
		@@ -45,9 +25,7 @@ def main():
		version='%(prog)s pysen={version}'.format(version=version(full=True)))

		args = parser.parse_args()

		setup_logger(args.loglevel)

		convert_file(args.filename, args.out, compression=args.compression, overwrite=args.overwrite)

		if __name__ == "__main__":

pysen/inout/converters.py

0 → 100644

+100 −0

Original line number	Diff line number	Diff line
		"""
		File type translators
		"""
		import os.path
		import logging

		from ..revision import version as version_info

		from .hdf import HdfConverter, read_nexus_echo
		from .reader import read_echo, read_xyz, read_magnetic
		from .writer import EchoWriter, DiffractionWriter, TransmissionWriter
		from .utils import get_nsefiletype, is_file_newer

		def _make_outfilename(filename, filetype=None, outdir='.'):
		""" """
		outfile, _ = os.path.splitext(filename)
		if 'nexus' in filetype:
		outfile, _ = os.path.splitext(outfile)
		outfile = os.path.join(outdir, outfile + ".h5")
		#print(f"\noutfile={outfile}")
		return outfile

		def convert_to_hdf(filename, outdir, **kwargs):
		"""convert .echo, xyz analysis .xyz or (old) magnetic .dat file into an HDF5 file

		@note:
		will not overwrite the .h5 if it exists
		and it is newer than the .echo file

		"""
		data_reader = { 'echo' : read_echo,
		'xyz' : read_xyz,
		'magnetic' : read_magnetic,
		'nexus_echo' : read_nexus_echo,
		#
		'application/nse-echo' : read_echo,
		'application/nse-xyz' : read_xyz,
		'application/nexus-nse-echo' : read_nexus_echo,
		}

		data_type = kwargs.pop('data_type', None)
		mode = kwargs.pop('mode', 'w')
		schema = kwargs.pop('schema', None)
		compression = kwargs.pop('compression', None)
		overwrite = kwargs.get('overwrite', False)

		_log = logging.getLogger()
		# outfile
		filetype = get_nsefiletype(filename)
		basename = os.path.basename(filename)
		outfile = _make_outfilename(basename, outdir=outdir, filetype=filetype)

		_log.info('converting %s to HDF5', basename)
		if not overwrite and is_file_newer(outfile, filename):
		_log.info('file %s is newer than %s', outfile, filename)
		return outfile

		if data_type is None: # guess data type based on file
		reader = data_reader.get(filetype)
		else: # data type given
		reader = data_reader.get(data_type)
		data = reader(filename)
		converter = HdfConverter(outfile, mode=mode, version=version_info())
		converter.write(data, compression=compression, schema=schema)
		return outfile

		def convert_file1(filenames, outdir, **kwargs):
		"""convert a TACO/NeXus file into an pysen/HDF file """
		log = logging.getLogger()
		for filename in filenames:
		basename = os.path.basename(filename)
		try:
		outfile = convert_to_hdf(filename, outdir, **kwargs)
		log.info('converted %s to %s', basename, outfile)
		except RuntimeError as exc:
		log.warning("error converting %s: %s", basename, exc)
		except OSError as exc:
		log.warning("error converting %s: %s", filename, exc.strerror)



		WriterClassType = { 'application/nexus-nse-echo' : EchoWriter,
		'application/nexus-nse-diffraction' : DiffractionWriter,
		'application/nexus-nse-transmission' : TransmissionWriter }
		def convert_file2(filenames, outdir, **kwargs):
		"convert file"
		log = logging.getLogger()
		for filename in filenames:
		if not os.path.exists(filename):
		log.warning("file '%s' does not exist", filename)
		continue
		file_type = get_nsefiletype(filename)
		writer_class = WriterClassType.get(file_type, None)
		if not isinstance(writer_class, type):
		log.warning("file '%s' unknown file type %s", filename, file_type)
		continue
		writer = writer_class()
		if not writer.read_nexus(filename, **kwargs):
		continue
		writer.save(outdir=outdir)

pysen/inout/hdf.py

+1 −72

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		"""Convert .echo to HDF5"""
		import os.path
		import logging

		import h5py
		import numpy as np


		from ..revision import version as version_info
		from ..iostrings import encode
		from .reader import read_echo, read_magnetic, read_xyz
		from .reader import read_echo
		from .writer import EchoWriter
		from .utils import get_nsefiletype, is_file_newer

		#FIXME: we need to work on the idea of a schema (or not???)
		SCHEMA_DEFAULT = {
		@@ -183,70 +179,3 @@ def read_nexus_echo(filename, outdir='.'):
		raise RuntimeError("Error reading NeXus file {filename}")
		echofile = echo.save(outdir)
		return read_echo(echofile)


		def _make_outfilename(filename, filetype=None, outdir='.'):
		""" """
		outfile, _ = os.path.splitext(filename)
		if 'nexus' in filetype:
		outfile, _ = os.path.splitext(outfile)
		outfile = os.path.join(outdir, outfile + ".h5")
		#print(f"\noutfile={outfile}")
		return outfile

		def convert_to_hdf(filename, outdir, **kwargs):
		"""convert .echo, xyz analysis .xyz or (old) magnetic .dat file into an HDF5 file

		@note:
		will not overwrite the .h5 if it exists
		and it is newer than the .echo file

		"""
		data_reader = { 'echo' : read_echo,
		'xyz' : read_xyz,
		'magnetic' : read_magnetic,
		'nexus_echo' : read_nexus_echo,
		#
		'application/nse-echo' : read_echo,
		'application/nse-xyz' : read_xyz,
		'application/nexus-nse-echo' : read_nexus_echo,
		}

		data_type = kwargs.pop('data_type', None)
		mode = kwargs.pop('mode', 'w')
		schema = kwargs.pop('schema', None)
		compression = kwargs.pop('compression', None)
		overwrite = kwargs.get('overwrite', False)

		_log = logging.getLogger()
		# outfile
		filetype = get_nsefiletype(filename)
		basename = os.path.basename(filename)
		outfile = _make_outfilename(basename, outdir=outdir, filetype=filetype)

		_log.info('converting %s to HDF5', basename)
		if not overwrite and is_file_newer(outfile, filename):
		_log.info('file %s is newer than %s', outfile, filename)
		return outfile

		if data_type is None: # guess data type based on file
		reader = data_reader.get(filetype)
		else: # data type given
		reader = data_reader.get(data_type)
		data = reader(filename)
		converter = HdfConverter(outfile, mode=mode, version=version_info())
		converter.write(data, compression=compression, schema=schema)
		return outfile

		#def xyz_to_hdf(filename, outdir, **kwargs):
		# """convert magnetic xyz up/down .xyz file into an HDF file """
		# mode = kwargs.pop('mode', 'w')
		# schema = kwargs.pop('schema', None)
		# compression = kwargs.pop('compression', None)
		#
		# outfile, _ = os.path.splitext(os.path.basename(filename))
		# outfile = os.path.join(outdir, outfile + ".h5")
		# data = read_xyz(filename)
		# converter = HdfConverter(outfile, mode=mode, version=version_info())
		# converter.write(data, compression=compression, schema=schema)
		# return outfile