Unverified Commit 62f7e6dc authored by Jose Borreguero's avatar Jose Borreguero Committed by GitHub
Browse files

json configuration input is corrupt (#251) 



* template file, extracting and substitue functions

Signed-off-by: default avatarJose Borreguero <borreguero@gmail.com>

* ensure template config is for CG1D

Signed-off-by: default avatarJose Borreguero <borreguero@gmail.com>

* Fix config dict not being updated properly

* Add tests for corrupted template config json

* Add unit tests for substitute_template in autoredux

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci



* Fix pre commit errors in test

* Fix docstring in extract_info_from_path

* Add exception handling for loading template json for autoredux

* Fix failing autoredux integration tests for template json

* Fix failing template config tests

Signed-off-by: default avatarJose Borreguero <borreguero@gmail.com>
Co-authored-by: default avatarCage G E <gzi@analysis-node18.sns.gov>
Co-authored-by: default avatarpre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
parent 60add46f

scripts/reduce_CG1D.py

+11 −2
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ from imars3d.backend.dataio.config import save_config 
from imars3d.backend import auto_reduction_ready

from imars3d.backend import load_template_config

from imars3d.backend import extract_info_from_path

from imars3d.backend import substitute_template

from imars3d.backend.autoredux import logger as logger_autoredux

from imars3d.backend.workflow.engine import WorkflowEngineAuto, WorkflowEngineError, WorkflowEngineExitCodes
@@ -83,16 +84,24 @@ def main(inputfile: Union[str, Path], outputdir: Union[str, Path]) -> int: 
    # step 1: load the template configuration file to memory

    try:

        config_path = _find_template_config()

        config_dict = load_template_config(config_path)

    except FileNotFoundError as e:

        logger.error(str(e))

        return ERROR_GENERAL

    config_dict = load_template_config(config_path)



    # step 2: extract info from inputfile

    update_dict = extract_info_from_path(str(inputfile))

    assert update_dict["instrument"] == "CG1D", "Instrument is not CG1D"

    update_dict["outputdir"] = str(outputdir)

    update_dict["workingdir"] = str(outputdir)



    # step 3: update the dict and save dict to disk

    config_dict.update(update_dict)

    try:

        config_dict = substitute_template(config_dict, update_dict)

    except Exception as e:

        logger.error(str(e))

        return ERROR_GENERAL



    # save config file to working directory

    # NOTE:

    #  i.e. ironman_20221108_154015.json

scripts/reduce_CG1D_config_template.json

0 → 100644
+116 −0
Original line number Diff line number Diff line 
{

    "facility": "$facility",

    "instrument": "$instrument",

    "ipts": "$ipts",

    "name": "reduce_$instrument",

    "workingdir": "$workingdir",

    "outputdir": "$outputdir",

    "tasks": [{

            "name": "load_data",

            "function": "imars3d.backend.dataio.data.load_data",

            "inputs": {

                "ct_dir": "$ctdir",

                "ob_dir": "$obdir",

                "dc_dir": "$dcdir",

                "ct_fnmatch": "*.tiff",

                "ob_fnmatch": "*.tiff",

                "dc_fnmatch": "*.tiff"

            },

            "outputs": ["ct", "ob", "dc", "rot_angles"]

        },

        {

            "name": "crop-ct",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "ct",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["ct"]

        },

        {

            "name": "crop-ob",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "ob",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["ob"]

        },

        {

            "name": "crop-dc",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "dc",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["dc"]

        },

        {

            "name": "gamma_filter",

            "function": "imars3d.backend.corrections.gamma_filter.gamma_filter",

            "inputs": {

                "arrays" : "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "normalization",

            "function": "imars3d.backend.preparation.normalization.normalization",

            "inputs": {

                "arrays": "ct",

                "flats": "ob",

                "darks": "dc"

            },

            "outputs": ["ct"]

        },

        {

            "name": "denoise",

            "function": "imars3d.backend.corrections.denoise.denoise",

            "inputs": {

                "arrays": "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "remove_ring_artifact",

            "function": "imars3d.backend.corrections.ring_removal.remove_ring_artifact",

            "inputs": {

                "arrays": "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "find_rotation_center",

            "function": "imars3d.backend.diagnostics.rotation.find_rotation_center",

            "inputs": {

                "arrays": "ct",

                "angles": "rot_angles",

                "in_degrees": true,

                "atol_deg": 0.5

            },

            "outputs": ["rot_center"]

        },

        {

            "name": "recon",

            "function": "imars3d.backend.reconstruction.recon",

            "inputs": {

                "arrays": "ct",

                "theta": "rot_angles",

                "center": "rot_center",

                "is_radians": false,

                "perform_minus_log": true

            },

            "outputs": ["result"]

        },

        {

            "name": "save_data",

            "function": "imars3d.backend.dataio.data.save_data",

            "inputs": {

                "data": "result",

                "filename": "test",

                "outputdir" : "outputdir"

            },

            "outputs": []

        }

    ]

}

src/imars3d/backend/__init__.py

+1 −0
Original line number Diff line number Diff line
@@ -2,3 +2,4 @@ 
from .autoredux import auto_reduction_ready  # noqa: F401

from .autoredux import load_template_config  # noqa: F401

from .autoredux import extract_info_from_path  # noqa: F401

from .autoredux import substitute_template  # noqa: F401

src/imars3d/backend/autoredux.py

+76 −11
Original line number Diff line number Diff line
@@ -4,17 +4,15 @@ 
# standard imports

import json

import logging

import re

from pathlib import Path

import os

import re

from string import Template

from typing import Union



# parent logger for all loggers instantiated in the auto-reduction scripts

logger = logging.getLogger(__name__)



facility_position = -6

instrument_position = -5

experiment_position = -4





def auto_reduction_ready(data_file: Union[str, Path]) -> bool:

    """
@@ -79,26 +77,93 @@ def load_template_config(config_path: Union[str, Path]) -> dict: 
        return config





def extract_info_from_path(data_file: str) -> dict:

def extract_info_from_path(data_file: Union[str, Path]) -> dict:

    """

    Extract information from the data file path.



    The data file path must conform to the following directory hierarchy

        /FACILITY/INSTRUMENT/IPTS/raw/[PREPATH]/ct_scans/[SUBPATH]/IMAGENAME.tiff

    PREPATH and SUBPATH are optional and can encompass one or more directories.



    Examples

    --------

    /HFIR/CG1D/IPTS-25777/raw/ct_scans/20191029_ironman_small_0070_000_000_0002.tiff

    /HFIR/CG1D/IPTS-25777/raw/2019_Oct/ct_scans/20191029_ironman_small_0070_000_000_0002.tiff

    /HFIR/CG1D/IPTS-25777/raw/2019_Oct/29/ct_scans/20191029_ironman_small_0070_000_000_0002.tiff

    /HFIR/CG1D/IPTS-25777/raw/2019_Oct/29/ct_scans/iron_man/XYZ/20191029_ironman_small_0070_000_000_0002.tiff



    Parameters

    ----------

    data_file

        The data file to extract information from.

        ex. /HFIR/CG1D/IPTS-25777/raw/ct_scans/iron_man/20191029_ironman_small_0070_000_000_0002.tiff

        The data file to extract information



    Returns

    -------

    dict

        The extracted information.

        The extracted information with the following keys:

        facility, instrument, ipts, prepath, subpath

    """

    extracted_data = data_file.split("/")

    # Assuming the path is an absolute path

    facility_position = 1

    instrument_position = 2

    experiment_position = 3

    raw_position = 4



    extracted_data = Path(data_file).parts

    assert extracted_data[0] == os.path.sep, f"Path {str(data_file)} is not an absolute path"



    # Extract FACILITY, INSTRUMENT, and IPTS

    data_dict = {}

    # index from right to be agnostic of root

    data_dict["facility"] = extracted_data[facility_position]  # "HFIR"

    data_dict["instrument"] = extracted_data[instrument_position]  # "CG1D"

    data_dict["ipts"] = extracted_data[experiment_position]  # "IPTS-25777"



    # Locate the index of the radiographs directory in list `extracted_data`

    ct_position = -1

    for i in range(1 + raw_position, len(extracted_data)):

        if extracted_data[i] in ["ct_scans", "ct", "radiographs"]:

            ct_position = i

            break

    assert ct_position > raw_position, "No radiographs directory found"

    data_dict["radiograph"] = extracted_data[ct_position]



    # Extract PREPATH and SUBPATH

    data_dict["prepath"] = os.path.sep.join(extracted_data[1 + raw_position : ct_position])

    data_dict["subpath"] = os.path.sep.join(extracted_data[1 + ct_position : -1])



    logger.info("Extract information from data file path.")

    return data_dict





def substitute_template(config: dict, values: dict) -> dict:

    r"""Update the template configuration with actual values.



    Parameters

    ----------

    config

        dictionary resulting from loading the template JSON configuration file

    values

        dictionary resulting from scanning the path to the input radiograph

        with function `extract_info_from_path`



    Returns

    -------

        dictionary with template keywords substituted with actual values

    """

    # Compose ct_dir, ob_dir, and dc_dir as PREPATH/IMAGE_TYPE/SUBPATH

    raw_path = Path("/")

    for subdir in ("facility", "instrument", "ipts"):

        raw_path = raw_path / values[subdir]

    pre_path = raw_path / "raw" / values["prepath"]

    values.update(

        {

            "ctdir": str(pre_path / values["radiograph"] / values["subpath"]),

            "obdir": str(pre_path / "ob" / values["subpath"]),

            "dcdir": str(pre_path / "df" / values["subpath"]),

        }

    )

    assert {"facility", "instrument", "ipts", "name", "workingdir", "outputdir", "tasks"}.issubset(

        set(config.keys())

    ), "Config template dict is missing keys."

    template = Template(json.dumps(config))

    return json.loads(template.substitute(**values))

tests/data/json/reduce_config_template_corrupt.json

0 → 100644
+116 −0
Original line number Diff line number Diff line 
{

    "facility": "$facility",

    "instrument": "$instrument",

    "BAD_ipts": "$ipts",

    "name": "reduce_$instrument",

    "workingdir": "$workingdir",

    "outputdir": "$outputdir",

    "tasks": [{

            "name": "BAD_load_data",

            "function": "imars3d.backend.dataio.data.load_data",

            "inputs": {

                "ct_dir": "$ctdir",

                "ob_dir": "$obdir",

                "dc_dir": "$dcdir",

                "ct_fnmatch": "*.tiff",

                "ob_fnmatch": "*.tiff",

                "dc_fnmatch": "*.tiff"

            },

            "outputs": ["ct", "ob", "dc", "rot_angles"]

        },

        {

            "name": "BAD_crop-ct",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "ct",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["ct"]

        },

        {

            "name": "crop-ob",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "ob",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["ob"]

        },

        {

            "name": "crop-dc",

            "function": "imars3d.backend.morph.crop.crop",

            "inputs": {

                "arrays": "dc",

                "crop_limit": [552, 1494, 771, 1296]

            },

            "outputs": ["dc"]

        },

        {

            "name": "gamma_filter",

            "function": "imars3d.backend.corrections.gamma_filter.gamma_filter",

            "inputs": {

                "arrays" : "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "normalization",

            "function": "imars3d.backend.preparation.normalization.normalization",

            "inputs": {

                "arrays": "ct",

                "flats": "ob",

                "darks": "dc"

            },

            "outputs": ["ct"]

        },

        {

            "name": "denoise",

            "function": "imars3d.backend.corrections.denoise.denoise",

            "inputs": {

                "arrays": "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "remove_ring_artifact",

            "function": "imars3d.backend.corrections.ring_removal.remove_ring_artifact",

            "inputs": {

                "arrays": "ct"

            },

            "outputs": ["ct"]

        },

        {

            "name": "find_rotation_center",

            "function": "imars3d.backend.diagnostics.rotation.find_rotation_center",

            "inputs": {

                "arrays": "ct",

                "angles": "rot_angles",

                "in_degrees": true,

                "atol_deg": 0.5

            },

            "outputs": ["rot_center"]

        },

        {

            "name": "recon",

            "function": "imars3d.backend.reconstruction.recon",

            "inputs": {

                "arrays": "ct",

                "theta": "rot_angles",

                "center": "rot_center",

                "is_radians": false,

                "perform_minus_log": true

            },

            "outputs": ["result"]

        },

        {

            "name": "save_data",

            "function": "imars3d.backend.dataio.data.save_data",

            "inputs": {

                "data": "result",

                "filename": "test",

                "outputdir" : "outputdir"

            },

            "outputs": []

        }

    ]

}