Unverified Commit 89441023 authored by Zhang, Chen's avatar Zhang, Chen Committed by GitHub
Browse files

Merge pull request #276 from ornlneutronimaging/loaddata_many_obdfdir 

extend load_data's ob_dir and df_dir to handle a list of folders
parents 0650f690 051c7ac4

src/imars3d/backend/dataio/data.py

+28 −2
Original line number Diff line number Diff line
@@ -42,6 +42,32 @@ logger = logging.getLogger(__name__) 
# }





class Foldernames(param.Foldername):

    r"""

    Parameter that can be set to a string specifying the path of a folder, or a list of such strings.



    The string(s) should be specified in UNIX style, but they will be

    returned in the format of the user's operating system.



    The specified path(s) can be absolute, or relative to either:



    * any of the paths specified in the search_paths attribute (if search_paths is not None);



    or



    * any of the paths searched by resolve_dir_path() (if search_paths is None).

    """



    def _resolve(self, paths):

        if isinstance(paths, (str, Path)):

            return super()._resolve(paths)

        elif isinstance(paths, (List, Tuple)):

            return [self._resolve(path) for path in paths]

        else:

            name = next(x for x in [self.name, self.label, "Foldernames parameter"] if x)

            raise ValueError(f"{name} must be a string or a list of strings")





class load_data(param.ParameterizedFunction):

    """

    Load data with given input.
@@ -100,8 +126,8 @@ class load_data(param.ParameterizedFunction): 
    dc_files = param.List(doc="list of all dc files to load")

    #

    ct_dir = param.Foldername(doc="radiograph directory")

    ob_dir = param.Foldername(doc="open beam directory")

    dc_dir = param.Foldername(doc="dark current directory")

    ob_dir = Foldernames(doc="open beam directory")

    dc_dir = Foldernames(doc="dark current directory")

    # NOTE: we need to provide a default value here as param.String default to "", which will

    #       not trigger dict.get(key, value) to get the value as "" is not None.

    ct_fnmatch = param.String(default="*", doc="fnmatch for selecting ct files from ct_dir")

tests/unit/backend/dataio/test_data.py

+36 −7
Original line number Diff line number Diff line
@@ -2,15 +2,21 @@ 
"""

Unit tests for backend data loading.

"""

import param



# package imports

from imars3d.backend.dataio.data import load_data

from imars3d.backend.dataio.data import save_data, save_checkpoint

from imars3d.backend.dataio.data import _forgiving_reader

from imars3d.backend.dataio.data import _load_images

from imars3d.backend.dataio.data import _load_by_file_list

from imars3d.backend.dataio.data import _get_filelist_by_dir

from imars3d.backend.dataio.data import _extract_rotation_angles

from imars3d.backend.dataio.data import (

    _extract_rotation_angles,

    _forgiving_reader,

    _get_filelist_by_dir,

    _load_by_file_list,

    _load_images,

    Foldernames,

    load_data,

    save_checkpoint,

    save_data,

)





# third party imports

import astropy.io.fits as fits
@@ -56,6 +62,29 @@ def data_fixture(tmpdir): 
    return generic_tiff, good_tiff, metadata_tiff, generic_fits





def test_Foldernames(tmpdir):

    class TestFoldernames(param.Parameterized):

        f = Foldernames(doc="input folders")



    # test wrong input

    with pytest.raises(ValueError) as e:

        TestFoldernames(f=open(tmpdir / "temp.txt", "w"))

    assert str(e.value) == "f must be a string or a list of strings"

    # test single directory

    tf = TestFoldernames(f=str(tmpdir))

    assert tf.f == str(tmpdir)

    tf = TestFoldernames(f=str(tmpdir))

    assert tf.f == str(tmpdir)

    # test multiple directories

    dir1, dir2 = tmpdir / "dir1", tmpdir / "dir2"

    dir1.mkdir()

    dir2.mkdir()

    tf = TestFoldernames(f=[str(dir1), str(dir2)])

    assert tf.f == [str(dir1), str(dir2)]

    tf = TestFoldernames(f=[dir1, dir2])

    assert tf.f == [str(dir1), str(dir2)]





@mock.patch("imars3d.backend.dataio.data._extract_rotation_angles")

@mock.patch("imars3d.backend.dataio.data._get_filelist_by_dir")

@mock.patch("imars3d.backend.dataio.data._load_by_file_list")