Unverified Commit fbd2683d authored by Zhang, Chen's avatar Zhang, Chen Committed by GitHub
Browse files

Merge pull request #253 from ornlneutronimaging/dxchange_refactor 

Refactor code to use central save functions
parents 8cf17b7b 416a8a91

src/imars3d/backend/dataio/data.py

+103 −13
Original line number Diff line number Diff line
@@ -475,41 +475,131 @@ def _extract_rotation_angles( 
    return rotation_angles





def _to_time_str(value: datetime) -> str:

    """

    Convert the supplied datetime to a formatted string.



    Parameters

    ----------

    value:

        datetime object to format correctly



    Returns

    -------

        The datetime as YYYYMMDDhhmm

    """

    return value.strftime("%Y%m%d%H%M")





def _save_data(filename: Path, data: np.ndarray, omegas: np.ndarray = None) -> None:

    if data is None:

        raise ValueError("Failed to supply data")

    logger.info(f'saving tiffs to "{filename.parent}"')



    # make sure the directory exists

    if not filename.parent.exists():

        filename.parent.mkdir(parents=True)

    # save the stack of tiffs

    dxchange.write_tiff_stack(data, fname=str(filename))



    # save the angles as a numpy object

    if omegas is not None:

        np.save(file=filename.parent / "omegas.npy", arr=omegas)





class save_data(param.ParameterizedFunction):

    """

    Save data with given input.



    The filenames will be

    ``<outputbase>/<name>_YYYYMMDDhhmm/<name>_####.tiff``

    where a canonical ``outputbase`` is ``/HFIR/CG1D/IPTS-23788/shared/processed_data/``.



    Parameters

    ----------

    data: Array

        array of data to save

    outputbase: Path

        where to save the output on disk.

        ``param.Foldername`` will warn if the directory does not already exist.

    name: str

        Used to name file of output, defaults to ``save_data``

    omegas: Array

        Optional for writing out the array of omega angles



    Returns

    -------

        The directory the files were actually saved in

    """



    data = param.Array(doc="Data to save", precedence=1)

    outputbase = param.Foldername(default="/tmp/", doc="radiograph directory")

    name = param.String(default="save_data", doc="name for the radiograph")

    omegas = param.Array(doc="Collection of omega angles")



    def __call__(self, **params):

        """Parse inputs and perform multiple dispatch."""

        # type bounds check via Parameter

        with param.logging_level("CRITICAL"):

            # do not complain about directories that don't exist

            _ = self.instance(**params)

        # sanitize arguments

        params = param.ParamOverrides(self, params)



        if params.data is None:

            raise ValueError("Did not supply data")



        save_dir = params.outputbase / f"{params.name}_{_to_time_str(datetime.now())}"



        # save the data as tiffs

        _save_data(filename=save_dir / params.name, data=params.data, omegas=params.omegas)



        return save_dir





class save_checkpoint(param.ParameterizedFunction):

    """

    Save current state to checkpoint in a datetime stamped directory name.



    The filenames will be

    ``<outputbase>/<name>_chkpt_YYYYMMDDhhmm/<name>_####.tiff``

    where a canonical ``outputbase`` is ``/HFIR/CG1D/IPTS-23788/shared/processed_data/``.



    Parameters

    ----------

    data: Array

        array of data to save

    outputdir: str

        where to save the output on disk

    filename: str

    outputbase: Path

        The parent directory of where to save the output on disk.

        ``param.Foldername`` will warn if the directory does not already exist.

    name: str

        Used to name file of output, defaults to output_{datetime}

    omegas: Array

        Optional for writing out the array of omega angles



    Returns

    -------

        None

        The directory the files were actually saved in

    """



    #

    data = param.Array(doc="Data to save", precedence=1)

    outputdir = param.Foldername(default="/tmp/", doc="radiograph directory")

    outputbase = param.Foldername(default="/tmp/", doc="directory checkpoint should exist in")



    filename = param.String(default="*", doc="fnmatch for selecting dc files from dc_dir")

    name = param.String(default="*", doc="name for the checkpoint")

    omegas = param.Array(doc="Collection of omega angles")



    def __call__(self, **params):

        """Parse inputs and perform multiple dispatch."""

        # type bounds check via Parameter

        with param.logging_level("CRITICAL"):

            # do not complain about directories that don't exist

            _ = self.instance(**params)

        # sanitize arguments

        params = param.ParamOverrides(self, params)



        if params.filename == "*":

            params.filename = "output_" + str(datetime.now())

        save_dir = params.outputbase / f"{params.name}_chkpt_{_to_time_str(datetime.now())}"



        self._save_data(params.data, params.outputdir + params.filename)

        # save the data as tiffs

        _save_data(filename=save_dir / params.name, data=params.data, omegas=params.omegas)



    def _save_data(self, data, filename):

        dxchange.write_tiff_stack(data, fname=filename)
 
        return save_dir

src/imars3d/ui/stages/preprocess.py

+2 −19
Original line number Diff line number Diff line
@@ -4,13 +4,11 @@ import param 
import panel as pn

import holoviews as hv

import numpy as np

import datetime

import os

import dxchange

from holoviews import streams

from holoviews import opts

from holoviews.operation.datashader import rasterize

from pathlib import Path

from imars3d.backend.dataio.data import save_checkpoint as imars_save_checkpoint

from imars3d.ui.widgets.gamma_filter import GammaFilter

from imars3d.ui.widgets.normalization import Normalization

from imars3d.ui.widgets.denoise import Denoise
@@ -100,22 +98,7 @@ class Preprocess(param.Parameterized): 
        if self.ct is None:

            pn.state.warning("No CT to save")

        else:

            # make dir

            chk_root = datetime.datetime.now().isoformat().replace(":", "_")

            savedirname = f"{self.temp_root}/{self.recn_name}/{chk_root}"

            os.makedirs(savedirname)

            # save the current CT

            dxchange.writer.write_tiff_stack(

                data=self.ct,

                fname=f"{savedirname}/chk.tiff",

                axis=0,

                digit=5,

            )

            # save omega list as well

            np.save(

                file=f"{savedirname}/omegas.py",

                arr=self.omegas,

            )

            imars_save_checkpoint(data=self.ct, outputbase=self.temp_root, name=self.recn_name, omegas=self.omegas)



    def cross_hair_view(self, x, y):

        """Cross hair view."""

src/imars3d/ui/stages/reconstruction.py

+4 −35
Original line number Diff line number Diff line
@@ -4,14 +4,13 @@ import param 
import panel as pn

import holoviews as hv

import numpy as np

import datetime

import os

import dxchange

from holoviews import streams

from holoviews import opts

from holoviews.operation.datashader import rasterize

from pathlib import Path

from imars3d.backend.reconstruction import recon

from imars3d.backend.dataio.data import save_checkpoint as imars_save_checkpoint

from imars3d.backend.dataio.data import save_data

from imars3d.ui.widgets.rotation import FindRotationCenter
@@ -131,22 +130,7 @@ class Reconstruction(param.Parameterized): 
        if self.ct is None:

            pn.state.warning("No CT to save")

        else:

            # make dir

            chk_root = datetime.datetime.now().isoformat().replace(":", "_")

            savedirname = f"{self.temp_root}/{self.recn_name}/{chk_root}"

            os.makedirs(savedirname)

            # save the current CT

            dxchange.writer.write_tiff_stack(

                data=self.ct,

                fname=f"{savedirname}/chk.tiff",

                axis=0,

                digit=5,

            )

            # save omega list as well

            np.save(

                file=f"{savedirname}/omegas.py",

                arr=self.omegas,

            )

            imars_save_checkpoint(data=self.ct, outputbase=self.temp_root, name=self.recn_name, omegas=self.omegas)



    @param.depends("recon_save", watch=True)

    def save_reconstruction_results(self):
@@ -154,22 +138,7 @@ class Reconstruction(param.Parameterized): 
        if self.recon is None:

            pn.state.warning("No reconstruction results to save")

        else:

            # make dir

            chk_root = datetime.datetime.now().isoformat().replace(":", "_")

            savedirname = f"{self.recn_root}/{self.recn_name}/{chk_root}"

            os.makedirs(savedirname)

            # save the current CT

            dxchange.writer.write_tiff_stack(

                data=self.recon,

                fname=f"{savedirname}/recon.tiff",

                axis=0,

                digit=5,

            )

            # save omega list as well

            np.save(

                file=f"{savedirname}/omegas.py",

                arr=self.omegas,

            )

            savedirname = save_data(data=self.ct, outputbase=self.recn_root, name=self.recn_name)

            # save the rotation center

            with open(f"{savedirname}/rot_center.txt", "w") as f_rotcnt:

                f_rotcnt.write(f"{self.rotation_center_finder.rot_center}")

src/imars3d/ui/stages/selectroi.py

+2 −19
Original line number Diff line number Diff line
@@ -4,13 +4,11 @@ import param 
import panel as pn

import holoviews as hv

import numpy as np

import datetime

import os

import dxchange

from holoviews import opts

from holoviews import streams

from pathlib import Path

from imars3d.backend.morph.crop import crop

from imars3d.backend.dataio.data import save_checkpoint as imars_save_checkpoint





class SelectROI(param.Parameterized):
@@ -102,22 +100,7 @@ class SelectROI(param.Parameterized): 
        if self.ct is None:

            pn.state.warning("No CT to save")

        else:

            # make dir

            chk_root = datetime.datetime.now().isoformat().replace(":", "_")

            savedirname = f"{self.temp_root}/{self.recn_name}/{chk_root}"

            os.makedirs(savedirname)

            # save the current CT

            dxchange.writer.write_tiff_stack(

                data=self.ct,

                fname=f"{savedirname}/chk.tiff",

                axis=0,

                digit=5,

            )

            # save omega list as well

            np.save(

                file=f"{savedirname}/omegas.py",

                arr=self.omegas,

            )

            imars_save_checkpoint(data=self.ct, outputbase=self.temp_root, name=self.recn_name, omegas=self.omegas)



    @param.depends("confirm_ROI_action", watch=True)

    def update_ROI(self):

tests/data/json/good_non_interactive_full.json

+2 −2
Original line number Diff line number Diff line
@@ -107,8 +107,8 @@ 
            "function": "imars3d.backend.dataio.data.save_data",

            "inputs": {

                "data": "result",

                "filename": "test",

                "outputdir" : "outputdir"

                "name": "testfull",

                "outputbase" : "outputdir"

            },

            "outputs": []

        }