From bc514caa38300f738a3bf1e026ea1628c40c58ca Mon Sep 17 00:00:00 2001 From: Cecilia Clark Date: Tue, 23 Sep 2025 13:26:49 -0400 Subject: [PATCH 1/4] feat: Initial documentation with example configs --- examples/multi-module/README-multi-modules.md | 31 ++++++ .../multi-module/modules-a-p-o-config.json | 99 +++++++++++++++++++ examples/multi-module/modules-p-o-config.json | 68 +++++++++++++ .../single-module/README-single-modules.md | 79 +++++++++++++++ examples/single-module/module-a-config.json | 43 ++++++++ examples/single-module/module-p-config.json | 46 +++++++++ 6 files changed, 366 insertions(+) create mode 100644 examples/multi-module/README-multi-modules.md create mode 100644 examples/multi-module/modules-a-p-o-config.json create mode 100644 examples/multi-module/modules-p-o-config.json create mode 100644 examples/single-module/README-single-modules.md create mode 100644 examples/single-module/module-a-config.json create mode 100644 examples/single-module/module-p-config.json diff --git a/examples/multi-module/README-multi-modules.md b/examples/multi-module/README-multi-modules.md new file mode 100644 index 0000000..c23a8cd --- /dev/null +++ b/examples/multi-module/README-multi-modules.md @@ -0,0 +1,31 @@ +# Examples of Multi-Module Configurations +> For geospatial data processing, there are two routes to take for multi-module processing: either with or without including Module A. If included, Module A will always be listed first in the multi-module workflow, followed by Module P and then finally Module O. + +## Understanding the Entire Config +Multi-module configurations follow the same overall structure as single module configurations, simply with more than one module listed. Please reference the `README-single-modules.md` before attempting a multi-module approach. + +The general structure of the configuration remains the same: + +- Specify the input directory or series of directories: + - There will still be the initial `input` value to include outside of the module definitions. This is generally going to be the directory (or specific subdirectories) of raw, level 1B imagery. +- Define the modules: + - Just as with a standalone configuration, users will need to identify all of the modules being used for processing, in the correct order and correct paths. +- Define each module's variables: + - All variables are still defined as previously mentioned in the `README-single-modules.md`. However, the only caveat with the multi-module implementation is aligning the input/source and output directories between modules. + - **The output of a given module will always be the same path as the input to the following module.** + - **All non-primary modules will have the same path for their source and output directories.** +- Follow the template: + - Just as with a standalone configuration, users can specify module arguments within each module's template. + +## General Comments +- The multimodule configuration will provide outputs at each stage of processing, which are used as input to subsequent stages. This design is meant to support easier troubleshooting between modules, if needed. +- As imagery is processed, output files will be appended with the name of the processing module. Therefore, imagery processed with Modules A, P, and O will have filenames ending in `MODA_MODP_MODO`. + - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. +- For multi-module processing, combinations can **only** be made in these two orders: + > Module A -> Module P -> Module O + > Module P -> Module O + - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. +- For more thorough discussions on variables and structure for individual modules, please browse the README for each: + - [Module A](https://code.ornl.gov/GSHS/common/pipe/module-a/-/blob/main/README.md) + - [Module P](https://code.ornl.gov/GSHS/common/pipe/module-p/-/blob/main/README.md) + - [Module O](https://code.ornl.gov/GSHS/common/pipe/module-o/-/blob/main/README.md) \ No newline at end of file diff --git a/examples/multi-module/modules-a-p-o-config.json b/examples/multi-module/modules-a-p-o-config.json new file mode 100644 index 0000000..67f8ee5 --- /dev/null +++ b/examples/multi-module/modules-a-p-o-config.json @@ -0,0 +1,99 @@ +{ + "version": "0.0.9", + "description": "Test workflow configuration", + "requirements": [ + "python", + "conda" + ], + "keywords": [], + "input": + "/path/to/input/directory", + "modules": [ + { + "name": "Module A (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-a", + "variables": { + "OUTPUT_DIRECTORY": "/path/to/output/directory", + "METHOD": "boa_reflectance", + "PROFILE": "urban" + }, + "template": { + "command": "python", + "environment": { + "name": "module-a", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--input_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--profile", + "{{ PROFILE }}" + ] + } + }, + { + "name": "Module P (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-p", + "variables": { + "SOURCE_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/path/to/output/directory", + "METHOD": "nn_diffuse", + "MODULE_LIST": "MODA, MODP, MODO" + }, + "template": { + "command": "python", + "environment": { + "name": "module-p", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ SOURCE_DIRECTORY }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--module_list", + "{{ MODULE_LIST }}" + ] + } + }, + { + "name": "Module O (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-o", + "variables": { + "SOURCE_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/path/to/output/directory" + }, + "template": { + "command": "python", + "environment": { + "name": "module-o", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ SOURCE_DIRECTORY }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}" + ] + } + } + ] +} \ No newline at end of file diff --git a/examples/multi-module/modules-p-o-config.json b/examples/multi-module/modules-p-o-config.json new file mode 100644 index 0000000..c5e202e --- /dev/null +++ b/examples/multi-module/modules-p-o-config.json @@ -0,0 +1,68 @@ +{ + "version": "0.0.9", + "description": "Test workflow configuration", + "requirements": [ + "python", + "conda" + ], + "keywords": [], + "input": + "/path/to/input/directory", + "modules": [ + { + "name": "Module P (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-p", + "variables": { + "OUTPUT_DIRECTORY": "/path/to/output/directory", + "METHOD": "nn_diffuse", + "MODULE_LIST": "MODP, MODO" + }, + "template": { + "command": "python", + "environment": { + "name": "module-p", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--module_list", + "{{ MODULE_LIST }}" + ] + } + }, + { + "name": "Module O (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-o", + "variables": { + "SOURCE_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/path/to/output/directory" + }, + "template": { + "command": "python", + "environment": { + "name": "module-o", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ SOURCE_DIRECTORY }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}" + ] + } + } + ] +} \ No newline at end of file diff --git a/examples/single-module/README-single-modules.md b/examples/single-module/README-single-modules.md new file mode 100644 index 0000000..70e1a05 --- /dev/null +++ b/examples/single-module/README-single-modules.md @@ -0,0 +1,79 @@ +# Examples of Single Module Configurations +> For geospatial data processing, Modules A and P are available to run as standalone modules. Module O currently requires Module P processing first, and therefore cannot be standalone. See the multi-module configuration options to incorporate Module O processing. + +## Understanding the Entire Config +Single module configurations are relatively straightforward, as there are no variables to align between multiple modules. Instead, users simply need to modify four main parts of the config based on their specific processing needs. + +- Specify the input directory or series of directories: + - All standalone modules will use the `input` value to parse through imagery in parallel. + - In `module-a-config.json`, there is a single directory listed. PIPE will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/input/directory" + ``` + - In `module-p-config.json`, there are multiple directories listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to PIPE these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ] + ``` + - Modules A and P can both use either of these `input` configurations; they are not limited to strictly following the examples shown here. These input variables are only different between example `config.json` files to showcase the variation in `input` options. +- Define the module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs PIPE from where to install the module environment. For example, if a user has PIPE installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-a" + ``` +- Define the module variables: + - Variables are unique to each module (details below). + - For standalone modules specifically, pay close attention to the `input`: users do **not** need to specify `input` as a variable within the module. Rather, the `input` is noted in the `template`. +- Speaking of the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module A: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-a", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--input_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--profile", + "{{ PROFILE }}" + ] + } + ``` + - **NOTE**: + - For standalone modules the `input_directory` for Module A and `source_directory` for Module P will **always** be `INPUT` in the template. + +## Module Variables +While Module O is not an option as a standalone module, the variable definitions are included here as well for continuity. For more thorough discussions on variables and structure for individual modules, please browse the README for each: +- [Module A](https://code.ornl.gov/GSHS/common/pipe/module-a/-/blob/main/README.md) +- [Module P](https://code.ornl.gov/GSHS/common/pipe/module-p/-/blob/main/README.md) +- [Module O](https://code.ornl.gov/GSHS/common/pipe/module-o/-/blob/main/README.md) + +### Module A: ***A***tmopsheric Correction +- `input_directory`: the directory containing the raw, Level 1B Maxar imagery. Module A currently expects a Maxar data directory structure. Because Module A is either run as a standalone module or always the first in a multi-module sequence, this argument will always be `INPUT` in the template. +- `output_directory`: the directory in which to store the output of Module A processing. +- `method`: specification for users to define if they want the output to contain top-of-atmosphere reflectance (`toa_reflectance`) or bottom-of-atmosphere reflectance (`boa_reflectance`). For the best representation of true surface reflectance (e.g., the removal of the blue effects of the atmosphere), users should select the latter. +- `profile`: specification for users to define the aersol profile selected in the Py6S model. Module A is currently optimized for `urban` applications, so consider using `maritime` sparingly. + - Future work for Module A will incorporate the maritime-optimized atmospheric correction efforts led by Matt McCarthy. + +### Module P: ***P***ansharpening +- `source_directory`: the directory containing the raw, Level 1B Maxar imagery (or the output of Module A for multi-module implementations). When Module P is run as a standalone module, this argument will be `INPUT` in the template. For multi-module configurations, this argument should match the `output_directory` of Module A. +- `output_directory`: the directory in which to store the output of Module P processing. For multi-module configurations, this argument should match both the `source_directory` for Module P and the `output_directory` of Module A. +- `method`: specification for users to define a pansharpening method. Currently, only `nn_diffuse` is supported. +- `module_list`: specification for users to identify all modules used in the given configuration. The presence, or absence, of the string `MODA` in this argument determines how the input directories are processed, as Module P can be used standalone (where input is a Maxar directory) or as a multi-module configuration (where input is the output of Module A). Options for this argument are: 'MODP', 'MODP, MODO', 'MODA, MODP', 'MODA, MODP, MODO'. + +### Module O: ***O***rthorectification +- 'source_directory': the directory containing the output of Module P. This argument should match the `output_directory` of Module P, as Module O is only supported in mutli-module configurations. +- 'output_directory': the directory in which to store the output of Module O processing. This argument should match both the `source_directory` for Module O and the `output_directory` of Module P, as Module O is only supported in multi-module configurations. \ No newline at end of file diff --git a/examples/single-module/module-a-config.json b/examples/single-module/module-a-config.json new file mode 100644 index 0000000..4c6b205 --- /dev/null +++ b/examples/single-module/module-a-config.json @@ -0,0 +1,43 @@ +{ + "version": "0.0.9", + "description": "Test workflow configuration", + "requirements": [ + "python", + "conda" + ], + "keywords": [], + "input": + "/path/to/input/directory", + "modules": [ + { + "name": "Module A (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-a", + "variables": { + "OUTPUT_DIRECTORY": "/path/to/output/directory", + "METHOD": "boa_reflectance", + "PROFILE": "urban" + }, + "template": { + "command": "python", + "environment": { + "name": "module-a", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--input_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--profile", + "{{ PROFILE }}" + ] + } + } + ] +} \ No newline at end of file diff --git a/examples/single-module/module-p-config.json b/examples/single-module/module-p-config.json new file mode 100644 index 0000000..4fb965f --- /dev/null +++ b/examples/single-module/module-p-config.json @@ -0,0 +1,46 @@ +{ + "version": "0.0.9", + "description": "Test workflow configuration", + "requirements": [ + "python", + "conda" + ], + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ], + "modules": [ + { + "name": "Module P (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-p", + "variables": { + "OUTPUT_DIRECTORY": "/path/to/output/directory", + "METHOD": "nn_diffuse", + "MODULE_LIST": "MODP" + }, + "template": { + "command": "python", + "environment": { + "name": "module-p", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--module_list", + "{{ MODULE_LIST }}" + ] + } + } + ] +} \ No newline at end of file -- GitLab From 51832ea03d66e1320d05254fe779643885313227 Mon Sep 17 00:00:00 2001 From: Jason Wohlgemuth Date: Thu, 2 Oct 2025 14:09:40 -0400 Subject: [PATCH 2/4] docs: Rename examples README files so GitLab will display them --- examples/multi-module/{README-multi-modules.md => README.md} | 0 examples/single-module/{README-single-modules.md => README.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename examples/multi-module/{README-multi-modules.md => README.md} (100%) rename examples/single-module/{README-single-modules.md => README.md} (100%) diff --git a/examples/multi-module/README-multi-modules.md b/examples/multi-module/README.md similarity index 100% rename from examples/multi-module/README-multi-modules.md rename to examples/multi-module/README.md diff --git a/examples/single-module/README-single-modules.md b/examples/single-module/README.md similarity index 100% rename from examples/single-module/README-single-modules.md rename to examples/single-module/README.md -- GitLab From e14aba02602f5c333e65dbd942ac4f8e9a785d25 Mon Sep 17 00:00:00 2001 From: Cecilia Clark Date: Mon, 20 Oct 2025 17:49:00 -0400 Subject: [PATCH 3/4] fix: Update file structure, separate content --- .../README.md | 102 ++++++++++++++++++ .../config.json} | 0 examples/atmosphericcorrection/README.md | 70 ++++++++++++ .../config.json} | 0 examples/multi-module/README.md | 31 ------ examples/orthorectification/README.md | 63 +++++++++++ examples/orthorectification/config.json | 40 +++++++ .../README.md | 94 ++++++++++++++++ .../config.json} | 0 examples/pansharpening/README.md | 69 ++++++++++++ .../config.json} | 0 examples/single-module/README.md | 79 -------------- 12 files changed, 438 insertions(+), 110 deletions(-) create mode 100644 examples/atmosphericcorrection-pansharpening-orthorectification/README.md rename examples/{multi-module/modules-a-p-o-config.json => atmosphericcorrection-pansharpening-orthorectification/config.json} (100%) create mode 100644 examples/atmosphericcorrection/README.md rename examples/{single-module/module-a-config.json => atmosphericcorrection/config.json} (100%) delete mode 100644 examples/multi-module/README.md create mode 100644 examples/orthorectification/README.md create mode 100644 examples/orthorectification/config.json create mode 100644 examples/pansharpening-orthorectification/README.md rename examples/{multi-module/modules-p-o-config.json => pansharpening-orthorectification/config.json} (100%) create mode 100644 examples/pansharpening/README.md rename examples/{single-module/module-p-config.json => pansharpening/config.json} (100%) delete mode 100644 examples/single-module/README.md diff --git a/examples/atmosphericcorrection-pansharpening-orthorectification/README.md b/examples/atmosphericcorrection-pansharpening-orthorectification/README.md new file mode 100644 index 0000000..8747480 --- /dev/null +++ b/examples/atmosphericcorrection-pansharpening-orthorectification/README.md @@ -0,0 +1,102 @@ +# Configuration for: Atmospheric Correction, Pansharpening, and Orthorectification + +## Breaking Down the Configuration +- Specify the input directory or series of directories: + - Use the `input` value to parse through imagery in parallel. + - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/input/directory" + ``` + - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ] + ``` +- Define each module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-a" + ``` +- Define each module's variables: + - Variables are unique to each module (detailed below). +- Simply follow the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module A: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-a", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--input_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--profile", + "{{ PROFILE }}" + ] + } + ``` + +## Module Variables +For more thorough discussions on variables and overall structure, along with a link to technical documentation, please browse the README: +- [Module A](https://code-int.ornl.gov/gshs/common/imagery-processing/module-a/-/blob/main/README.md) +- [Module P](https://code-int.ornl.gov/gshs/common/imagery-processing/module-p/-/blob/main/README.md) +- [Module O](https://code-int.ornl.gov/gshs/common/imagery-processing/module-o/-/blob/main/README.md) + +### Module A: ***A***tmopsheric Correction +- `input_directory`: the directory containing the raw, Level 1B Maxar imagery. Module A currently expects a Maxar data directory structure. This argument will always be `INPUT` in the template. +- `output_directory`: the directory in which to store the output of Module A processing. +- `method`: specification for users to define if they want the output to contain top-of-atmosphere reflectance (`toa_reflectance`) or bottom-of-atmosphere reflectance (`boa_reflectance`). For the best representation of true surface reflectance (e.g., the removal of the blue effects of the atmosphere), users should select the latter. +- `profile`: specification for users to define the aersol profile selected in the Py6S model. Module A is currently optimized for `urban` applications, so consider using `maritime` sparingly. + - Future work for Module A will incorporate the maritime-optimized atmospheric correction efforts led by Matt McCarthy. + +### Module P: ***P***ansharpening +- `source_directory`: the directory containing the output of Module A. This argument should match the `output_directory` of Module A. +- `output_directory`: the directory in which to store the output of Module P processing. This argument should match both the `source_directory` for Module P and the `output_directory` of Module A. +- `method`: specification for users to define a pansharpening method. Currently, only `nn_diffuse` is supported. +- `module_list`: specification for users to identify all modules used in the given configuration. The presence, or absence, of the string `MODA` in this argument determines how the input directories are processed. In this example `config.json`, users would specify Modules A, P, and O. Options for this argument are: 'MODP', 'MODP, MODO', 'MODA, MODP', 'MODA, MODP, MODO'. + +### Module O: ***O***rthorectification +- 'source_directory': the directory containing the output of Module P. This argument should match the `output_directory` of Module P. +- 'output_directory': the directory in which to store the output of Module O processing. This argument should match both the `source_directory` for Module O and the `output_directory` of Module P. + +## Running the Module +- To run the standalone module independently of any workflow: + ```bash + make run + ``` +- To run the module with Xylem, at maximum verbosity, using the `config.json`: + ```bash + xylem run -vvvv + ``` + +## General Notes for a Multi-Module Configuration +- The general structure of the configuration remains the same: + - Specify the input directory or series of directories: + - There will still be the initial `input` value to include outside of the module definitions. This is generally going to be the directory (or specific subdirectories) of raw, level 1B imagery. + - Define the modules: + - Just as with a standalone configuration, users will need to identify all of the modules being used for processing, in the correct order and correct paths. + - Define each module's variables: + - All variables are defined as above. However, the only caveat with the multi-module implementation is aligning the input/source and output directories between modules. + - **The output of a given module will always be the same path as the input to the following module.** + - **All non-primary modules will have the same path for their source and output directories.** + - Follow the template: + - Just as with a standalone configuration, users can specify module arguments within each module's template. +- The multi-module configuration will provide outputs at each stage of processing, which are used as input to subsequent stages. This design is meant to support easier troubleshooting between modules, if needed. +- As imagery is processed, output files will be appended with the name of the processing module. Therefore, imagery processed with Modules A, P, and O will have filenames ending in `MODA_MODP_MODO`. + - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. +- For multi-module processing, combinations can **only** be made in these two orders: + > Module A -> Module P -> Module O + > Module P -> Module O + - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. \ No newline at end of file diff --git a/examples/multi-module/modules-a-p-o-config.json b/examples/atmosphericcorrection-pansharpening-orthorectification/config.json similarity index 100% rename from examples/multi-module/modules-a-p-o-config.json rename to examples/atmosphericcorrection-pansharpening-orthorectification/config.json diff --git a/examples/atmosphericcorrection/README.md b/examples/atmosphericcorrection/README.md new file mode 100644 index 0000000..6edb433 --- /dev/null +++ b/examples/atmosphericcorrection/README.md @@ -0,0 +1,70 @@ +# Configuration for: Atmospheric Correction + +## Breaking Down the Configuration +- Specify the input directory or series of directories: + - Use the `input` value to parse through imagery in parallel. + - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/input/directory" + ``` + - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ] + ``` +- Define the module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-a" + ``` +- Define the module variables: + - Variables are unique to each module (detailed below). +- Simply follow the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module A: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-a", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--input_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--profile", + "{{ PROFILE }}" + ] + } + ``` + +## Module Variables +For more thorough discussions on variables and overall structure, along with a link to technical documentation, please browse the README: +- [Module A](https://code-int.ornl.gov/gshs/common/imagery-processing/module-a/-/blob/main/README.md) + +### Module A: ***A***tmopsheric Correction +- `input_directory`: the directory containing the raw, Level 1B Maxar imagery. Module A currently expects a Maxar data directory structure. Because Module A is either run as a standalone module or always the first in a multi-module sequence, this argument will always be `INPUT` in the template. +- `output_directory`: the directory in which to store the output of Module A processing. +- `method`: specification for users to define if they want the output to contain top-of-atmosphere reflectance (`toa_reflectance`) or bottom-of-atmosphere reflectance (`boa_reflectance`). For the best representation of true surface reflectance (e.g., the removal of the blue effects of the atmosphere), users should select the latter. +- `profile`: specification for users to define the aersol profile selected in the Py6S model. Module A is currently optimized for `urban` applications, so consider using `maritime` sparingly. + - Future work for Module A will incorporate the maritime-optimized atmospheric correction efforts led by Matt McCarthy. + +## Running the Module +- To run the standalone module independently of any workflow: + ```bash + make run + ``` +- To run the module with Xylem, at maximum verbosity, using the `config.json`: + ```bash + xylem run -vvvv + ``` \ No newline at end of file diff --git a/examples/single-module/module-a-config.json b/examples/atmosphericcorrection/config.json similarity index 100% rename from examples/single-module/module-a-config.json rename to examples/atmosphericcorrection/config.json diff --git a/examples/multi-module/README.md b/examples/multi-module/README.md deleted file mode 100644 index c23a8cd..0000000 --- a/examples/multi-module/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Examples of Multi-Module Configurations -> For geospatial data processing, there are two routes to take for multi-module processing: either with or without including Module A. If included, Module A will always be listed first in the multi-module workflow, followed by Module P and then finally Module O. - -## Understanding the Entire Config -Multi-module configurations follow the same overall structure as single module configurations, simply with more than one module listed. Please reference the `README-single-modules.md` before attempting a multi-module approach. - -The general structure of the configuration remains the same: - -- Specify the input directory or series of directories: - - There will still be the initial `input` value to include outside of the module definitions. This is generally going to be the directory (or specific subdirectories) of raw, level 1B imagery. -- Define the modules: - - Just as with a standalone configuration, users will need to identify all of the modules being used for processing, in the correct order and correct paths. -- Define each module's variables: - - All variables are still defined as previously mentioned in the `README-single-modules.md`. However, the only caveat with the multi-module implementation is aligning the input/source and output directories between modules. - - **The output of a given module will always be the same path as the input to the following module.** - - **All non-primary modules will have the same path for their source and output directories.** -- Follow the template: - - Just as with a standalone configuration, users can specify module arguments within each module's template. - -## General Comments -- The multimodule configuration will provide outputs at each stage of processing, which are used as input to subsequent stages. This design is meant to support easier troubleshooting between modules, if needed. -- As imagery is processed, output files will be appended with the name of the processing module. Therefore, imagery processed with Modules A, P, and O will have filenames ending in `MODA_MODP_MODO`. - - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. -- For multi-module processing, combinations can **only** be made in these two orders: - > Module A -> Module P -> Module O - > Module P -> Module O - - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. -- For more thorough discussions on variables and structure for individual modules, please browse the README for each: - - [Module A](https://code.ornl.gov/GSHS/common/pipe/module-a/-/blob/main/README.md) - - [Module P](https://code.ornl.gov/GSHS/common/pipe/module-p/-/blob/main/README.md) - - [Module O](https://code.ornl.gov/GSHS/common/pipe/module-o/-/blob/main/README.md) \ No newline at end of file diff --git a/examples/orthorectification/README.md b/examples/orthorectification/README.md new file mode 100644 index 0000000..bcc40c3 --- /dev/null +++ b/examples/orthorectification/README.md @@ -0,0 +1,63 @@ +# Configuration for: Orthorectification + +## Breaking Down the Configuration +- Specify the input directory or series of directories: + - Use the `input` value to parse through imagery in parallel. + - In the associated `config.json`, multiple directories are listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/output_MODP/order1", + "/path/to/output_MODP/order2", + "/path/to/output_MODP/order3", + "/path/to/output_MODP/order4" + ] + ``` + - A single directory may also be specified. Xylem will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/output_MODP" + ``` +- Define the module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-o" + ``` +- Define the module variables: + - Variables are unique to each module (detailed below). +- Simply follow the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module O: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-o", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}" + ] + } + ``` + +## Module Variables +For more thorough discussions on variables and overall structure, along with a link to technical documentation, please browse the README: +- [Module O](https://code-int.ornl.gov/gshs/common/imagery-processing/module-o/-/blob/main/README.md) + +### Module O: ***O***rthorectification +- 'source_directory': the directory containing the output of Module P. Module O can be run as a unique module following the example `config.json`. HOWEVER, the input MUST have already been processed through Module P. +- 'output_directory': the directory in which to store the output of Module O processing. This argument should match the `source_directory` for Module O. + +## Running the Module +- To run the standalone module independently of any workflow: + ```bash + make run + ``` +- To run the module with Xylem, at maximum verbosity, using the `config.json`: + ```bash + xylem run -vvvv + ``` \ No newline at end of file diff --git a/examples/orthorectification/config.json b/examples/orthorectification/config.json new file mode 100644 index 0000000..8723e8e --- /dev/null +++ b/examples/orthorectification/config.json @@ -0,0 +1,40 @@ +{ + "version": "0.0.9", + "description": "Test workflow configuration", + "requirements": [ + "python", + "conda" + ], + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ], + "modules": [ + { + "name": "Module O (local)", + "type": "script", + "programmingLanguage": "python", + "uri": "file:///path/to/module-o", + "variables": { + "OUTPUT_DIRECTORY": "/path/to/output/directory" + }, + "template": { + "command": "python", + "environment": { + "name": "module-o", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}" + ] + } + } + ] +} \ No newline at end of file diff --git a/examples/pansharpening-orthorectification/README.md b/examples/pansharpening-orthorectification/README.md new file mode 100644 index 0000000..baa0eec --- /dev/null +++ b/examples/pansharpening-orthorectification/README.md @@ -0,0 +1,94 @@ +# Configuration for: Pansharpening and Orthorectification + +## Breaking Down the Configuration +- Specify the input directory or series of directories: + - Use the `input` value to parse through imagery in parallel. + - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/input/directory" + ``` + - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ] + ``` +- Define each module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-p" + ``` +- Define each module's variables: + - Variables are unique to each module (detailed below). +- Simply follow the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module P: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-p", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--module_list", + "{{ MODULE_LIST }}" + ] + } + ``` + +## Module Variables +For more thorough discussions on variables and overall structure, along with a link to technical documentation, please browse the README: +- [Module P](https://code-int.ornl.gov/gshs/common/imagery-processing/module-p/-/blob/main/README.md) +- [Module O](https://code-int.ornl.gov/gshs/common/imagery-processing/module-o/-/blob/main/README.md) + +### Module P: ***P***ansharpening +- `source_directory`: the directory containing the raw, Level 1B Maxar imagery (or the output of Module A, if the images have been previously processed through Module A). +- `output_directory`: the directory in which to store the output of Module P processing. +- `method`: specification for users to define a pansharpening method. Currently, only `nn_diffuse` is supported. +- `module_list`: specification for users to identify all modules used in the given configuration. The presence, or absence, of the string `MODA` in this argument determines how the input directories are processed. In this example `config.json`, users would specify Modules P and O, assuming the imagery was not previously processing with Module A. Options for this argument are: 'MODP', 'MODP, MODO', 'MODA, MODP', 'MODA, MODP, MODO'. + +### Module O: ***O***rthorectification +- 'source_directory': the directory containing the output of Module P. This argument should match the `output_directory` of Module P. +- 'output_directory': the directory in which to store the output of Module O processing. This argument should match both the `source_directory` for Module O and the `output_directory` of Module P. + +## Running the Module +- To run the standalone module independently of any workflow: + ```bash + make run + ``` +- To run the module with Xylem, at maximum verbosity, using the `config.json`: + ```bash + xylem run -vvvv + ``` + +## General Notes for a Multi-Module Configuration +- The general structure of the configuration remains the same: + - Specify the input directory or series of directories: + - There will still be the initial `input` value to include outside of the module definitions. This is generally going to be the directory (or specific subdirectories) of raw, level 1B imagery. + - Define the modules: + - Just as with a standalone configuration, users will need to identify all of the modules being used for processing, in the correct order and correct paths. + - Define each module's variables: + - All variables are defined as above. However, the only caveat with the multi-module implementation is aligning the input/source and output directories between modules. + - **The output of a given module will always be the same path as the input to the following module.** + - **All non-primary modules will have the same path for their source and output directories.** + - Follow the template: + - Just as with a standalone configuration, users can specify module arguments within each module's template. +- The multi-module configuration will provide outputs at each stage of processing, which are used as input to subsequent stages. This design is meant to support easier troubleshooting between modules, if needed. +- As imagery is processed, output files will be appended with the name of the processing module. Therefore, imagery processed with Modules P and O will have filenames ending in `MODP_MODO`. + - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. +- For multi-module processing, combinations can **only** be made in these two orders: + > Module A -> Module P -> Module O + > Module P -> Module O + - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. \ No newline at end of file diff --git a/examples/multi-module/modules-p-o-config.json b/examples/pansharpening-orthorectification/config.json similarity index 100% rename from examples/multi-module/modules-p-o-config.json rename to examples/pansharpening-orthorectification/config.json diff --git a/examples/pansharpening/README.md b/examples/pansharpening/README.md new file mode 100644 index 0000000..11ca52d --- /dev/null +++ b/examples/pansharpening/README.md @@ -0,0 +1,69 @@ +# Configuration for: Pansharpening + +## Breaking Down the Configuration +- Specify the input directory or series of directories: + - Use the `input` value to parse through imagery in parallel. + - In the associated `config.json`, multiple directories are listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. + ```json + "input": [ + "/path/to/input/order1", + "/path/to/input/order2", + "/path/to/input/order3", + "/path/to/input/order4" + ] + ``` + - A single directory may also be specified. Xylem will parse through this main directory and concurrently process all available subdirectories. + ```json + "input": + "/path/to/input/directory" + ``` +- Define the module: + - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: + ```json + "uri": "file:///root/dev/module-p" + ``` +- Define the module variables: + - Variables are unique to each module (detailed below). +- Simply follow the `template`: + - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module P: + ```json + "template": { + "command": "python", + "environment": { + "name": "module-p", + "manager": "conda" + }, + "arguments": [ + "-m", + "lib", + "--source_directory", + "{{ INPUT }}", + "--output_directory", + "{{ OUTPUT_DIRECTORY }}", + "--method", + "{{ METHOD }}", + "--module_list", + "{{ MODULE_LIST }}" + ] + } + ``` + +## Module Variables +For more thorough discussions on variables and overall structure, along with a link to technical documentation, please browse the README: +- [Module P](https://code-int.ornl.gov/gshs/common/imagery-processing/module-p/-/blob/main/README.md) + +### Module P: ***P***ansharpening +- `source_directory`: the directory containing the raw, Level 1B Maxar imagery (or the output of Module A, if the images have been previously processed through Module A). When Module P is run as a standalone module, as with this `config.json`, this argument will be `INPUT` in the template. +- `output_directory`: the directory in which to store the output of Module P processing. +- `method`: specification for users to define a pansharpening method. Currently, only `nn_diffuse` is supported. +- `module_list`: specification for users to identify all modules used in the given configuration. The presence, or absence, of the string `MODA` in this argument determines how the input directories are processed. If the imagery has already been processed with Module A, include Module A on the list, even if you are only processing with Module P following the example `config.json`. Options for this argument are: 'MODP', 'MODP, MODO', 'MODA, MODP', 'MODA, MODP, MODO'. + +## Running the Module +- To run the standalone module independently of any workflow: + ```bash + make run + ``` +- To run the module with Xylem, at maximum verbosity, using the `config.json`: + ```bash + xylem run -vvvv + ``` \ No newline at end of file diff --git a/examples/single-module/module-p-config.json b/examples/pansharpening/config.json similarity index 100% rename from examples/single-module/module-p-config.json rename to examples/pansharpening/config.json diff --git a/examples/single-module/README.md b/examples/single-module/README.md deleted file mode 100644 index 70e1a05..0000000 --- a/examples/single-module/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Examples of Single Module Configurations -> For geospatial data processing, Modules A and P are available to run as standalone modules. Module O currently requires Module P processing first, and therefore cannot be standalone. See the multi-module configuration options to incorporate Module O processing. - -## Understanding the Entire Config -Single module configurations are relatively straightforward, as there are no variables to align between multiple modules. Instead, users simply need to modify four main parts of the config based on their specific processing needs. - -- Specify the input directory or series of directories: - - All standalone modules will use the `input` value to parse through imagery in parallel. - - In `module-a-config.json`, there is a single directory listed. PIPE will parse through this main directory and concurrently process all available subdirectories. - ```json - "input": - "/path/to/input/directory" - ``` - - In `module-p-config.json`, there are multiple directories listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to PIPE these specific directories should be concurrently processed. - ```json - "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" - ] - ``` - - Modules A and P can both use either of these `input` configurations; they are not limited to strictly following the examples shown here. These input variables are only different between example `config.json` files to showcase the variation in `input` options. -- Define the module: - - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs PIPE from where to install the module environment. For example, if a user has PIPE installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: - ```json - "uri": "file:///root/dev/module-a" - ``` -- Define the module variables: - - Variables are unique to each module (details below). - - For standalone modules specifically, pay close attention to the `input`: users do **not** need to specify `input` as a variable within the module. Rather, the `input` is noted in the `template`. -- Speaking of the `template`: - - This section is built from the module's `Makefile`. Here, users can specify arguments and associated variables. Below is an example template for Module A: - ```json - "template": { - "command": "python", - "environment": { - "name": "module-a", - "manager": "conda" - }, - "arguments": [ - "-m", - "lib", - "--input_directory", - "{{ INPUT }}", - "--output_directory", - "{{ OUTPUT_DIRECTORY }}", - "--method", - "{{ METHOD }}", - "--profile", - "{{ PROFILE }}" - ] - } - ``` - - **NOTE**: - - For standalone modules the `input_directory` for Module A and `source_directory` for Module P will **always** be `INPUT` in the template. - -## Module Variables -While Module O is not an option as a standalone module, the variable definitions are included here as well for continuity. For more thorough discussions on variables and structure for individual modules, please browse the README for each: -- [Module A](https://code.ornl.gov/GSHS/common/pipe/module-a/-/blob/main/README.md) -- [Module P](https://code.ornl.gov/GSHS/common/pipe/module-p/-/blob/main/README.md) -- [Module O](https://code.ornl.gov/GSHS/common/pipe/module-o/-/blob/main/README.md) - -### Module A: ***A***tmopsheric Correction -- `input_directory`: the directory containing the raw, Level 1B Maxar imagery. Module A currently expects a Maxar data directory structure. Because Module A is either run as a standalone module or always the first in a multi-module sequence, this argument will always be `INPUT` in the template. -- `output_directory`: the directory in which to store the output of Module A processing. -- `method`: specification for users to define if they want the output to contain top-of-atmosphere reflectance (`toa_reflectance`) or bottom-of-atmosphere reflectance (`boa_reflectance`). For the best representation of true surface reflectance (e.g., the removal of the blue effects of the atmosphere), users should select the latter. -- `profile`: specification for users to define the aersol profile selected in the Py6S model. Module A is currently optimized for `urban` applications, so consider using `maritime` sparingly. - - Future work for Module A will incorporate the maritime-optimized atmospheric correction efforts led by Matt McCarthy. - -### Module P: ***P***ansharpening -- `source_directory`: the directory containing the raw, Level 1B Maxar imagery (or the output of Module A for multi-module implementations). When Module P is run as a standalone module, this argument will be `INPUT` in the template. For multi-module configurations, this argument should match the `output_directory` of Module A. -- `output_directory`: the directory in which to store the output of Module P processing. For multi-module configurations, this argument should match both the `source_directory` for Module P and the `output_directory` of Module A. -- `method`: specification for users to define a pansharpening method. Currently, only `nn_diffuse` is supported. -- `module_list`: specification for users to identify all modules used in the given configuration. The presence, or absence, of the string `MODA` in this argument determines how the input directories are processed, as Module P can be used standalone (where input is a Maxar directory) or as a multi-module configuration (where input is the output of Module A). Options for this argument are: 'MODP', 'MODP, MODO', 'MODA, MODP', 'MODA, MODP, MODO'. - -### Module O: ***O***rthorectification -- 'source_directory': the directory containing the output of Module P. This argument should match the `output_directory` of Module P, as Module O is only supported in mutli-module configurations. -- 'output_directory': the directory in which to store the output of Module O processing. This argument should match both the `source_directory` for Module O and the `output_directory` of Module P, as Module O is only supported in multi-module configurations. \ No newline at end of file -- GitLab From dd28c251461aef44bbf7947994bd487e67641cac Mon Sep 17 00:00:00 2001 From: Cecilia Clark Date: Tue, 21 Oct 2025 14:57:26 -0400 Subject: [PATCH 4/4] fix: Add snakecase and test cases for examples --- .../README.md | 11 ++++++----- .../config.json | 12 ++++++------ .../README.md | 10 +++++----- .../config.json | 4 ++-- examples/orthorectification/README.md | 10 +++++----- examples/orthorectification/config.json | 10 +++++----- examples/pansharpening-orthorectification/README.md | 11 ++++++----- .../pansharpening-orthorectification/config.json | 8 ++++---- examples/pansharpening/README.md | 10 +++++----- examples/pansharpening/config.json | 10 +++++----- 10 files changed, 49 insertions(+), 47 deletions(-) rename examples/{atmosphericcorrection-pansharpening-orthorectification => atmospheric_correction-pansharpening-orthorectification}/README.md (96%) rename examples/{atmosphericcorrection-pansharpening-orthorectification => atmospheric_correction-pansharpening-orthorectification}/config.json (88%) rename examples/{atmosphericcorrection => atmospheric_correction}/README.md (92%) rename examples/{atmosphericcorrection => atmospheric_correction}/config.json (91%) diff --git a/examples/atmosphericcorrection-pansharpening-orthorectification/README.md b/examples/atmospheric_correction-pansharpening-orthorectification/README.md similarity index 96% rename from examples/atmosphericcorrection-pansharpening-orthorectification/README.md rename to examples/atmospheric_correction-pansharpening-orthorectification/README.md index 8747480..f0b772e 100644 --- a/examples/atmosphericcorrection-pansharpening-orthorectification/README.md +++ b/examples/atmospheric_correction-pansharpening-orthorectification/README.md @@ -6,15 +6,15 @@ - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. ```json "input": - "/path/to/input/directory" + "/data/test-cases" ``` - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. ```json "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ] ``` - Define each module: @@ -98,5 +98,6 @@ For more thorough discussions on variables and overall structure, along with a l - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. - For multi-module processing, combinations can **only** be made in these two orders: > Module A -> Module P -> Module O + > Module P -> Module O - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. \ No newline at end of file diff --git a/examples/atmosphericcorrection-pansharpening-orthorectification/config.json b/examples/atmospheric_correction-pansharpening-orthorectification/config.json similarity index 88% rename from examples/atmosphericcorrection-pansharpening-orthorectification/config.json rename to examples/atmospheric_correction-pansharpening-orthorectification/config.json index 67f8ee5..2f69578 100644 --- a/examples/atmosphericcorrection-pansharpening-orthorectification/config.json +++ b/examples/atmospheric_correction-pansharpening-orthorectification/config.json @@ -7,7 +7,7 @@ ], "keywords": [], "input": - "/path/to/input/directory", + "/data/test-cases", "modules": [ { "name": "Module A (local)", @@ -15,7 +15,7 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-a", "variables": { - "OUTPUT_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/data/output", "METHOD": "boa_reflectance", "PROFILE": "urban" }, @@ -45,8 +45,8 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-p", "variables": { - "SOURCE_DIRECTORY": "/path/to/output/directory", - "OUTPUT_DIRECTORY": "/path/to/output/directory", + "SOURCE_DIRECTORY": "/data/output", + "OUTPUT_DIRECTORY": "/data/output", "METHOD": "nn_diffuse", "MODULE_LIST": "MODA, MODP, MODO" }, @@ -76,8 +76,8 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-o", "variables": { - "SOURCE_DIRECTORY": "/path/to/output/directory", - "OUTPUT_DIRECTORY": "/path/to/output/directory" + "SOURCE_DIRECTORY": "/data/output", + "OUTPUT_DIRECTORY": "/data/output" }, "template": { "command": "python", diff --git a/examples/atmosphericcorrection/README.md b/examples/atmospheric_correction/README.md similarity index 92% rename from examples/atmosphericcorrection/README.md rename to examples/atmospheric_correction/README.md index 6edb433..04f5b4d 100644 --- a/examples/atmosphericcorrection/README.md +++ b/examples/atmospheric_correction/README.md @@ -6,15 +6,15 @@ - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. ```json "input": - "/path/to/input/directory" + "/data/test-cases" ``` - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. ```json "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ] ``` - Define the module: diff --git a/examples/atmosphericcorrection/config.json b/examples/atmospheric_correction/config.json similarity index 91% rename from examples/atmosphericcorrection/config.json rename to examples/atmospheric_correction/config.json index 4c6b205..9d2db55 100644 --- a/examples/atmosphericcorrection/config.json +++ b/examples/atmospheric_correction/config.json @@ -7,7 +7,7 @@ ], "keywords": [], "input": - "/path/to/input/directory", + "/data/test-cases", "modules": [ { "name": "Module A (local)", @@ -15,7 +15,7 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-a", "variables": { - "OUTPUT_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/data/output", "METHOD": "boa_reflectance", "PROFILE": "urban" }, diff --git a/examples/orthorectification/README.md b/examples/orthorectification/README.md index bcc40c3..cad6cb9 100644 --- a/examples/orthorectification/README.md +++ b/examples/orthorectification/README.md @@ -6,16 +6,16 @@ - In the associated `config.json`, multiple directories are listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. ```json "input": [ - "/path/to/output_MODP/order1", - "/path/to/output_MODP/order2", - "/path/to/output_MODP/order3", - "/path/to/output_MODP/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ] ``` - A single directory may also be specified. Xylem will parse through this main directory and concurrently process all available subdirectories. ```json "input": - "/path/to/output_MODP" + "/data/test-cases" ``` - Define the module: - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: diff --git a/examples/orthorectification/config.json b/examples/orthorectification/config.json index 8723e8e..88f49f5 100644 --- a/examples/orthorectification/config.json +++ b/examples/orthorectification/config.json @@ -6,10 +6,10 @@ "conda" ], "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ], "modules": [ { @@ -18,7 +18,7 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-o", "variables": { - "OUTPUT_DIRECTORY": "/path/to/output/directory" + "OUTPUT_DIRECTORY": "/data/output" }, "template": { "command": "python", diff --git a/examples/pansharpening-orthorectification/README.md b/examples/pansharpening-orthorectification/README.md index baa0eec..b2e6982 100644 --- a/examples/pansharpening-orthorectification/README.md +++ b/examples/pansharpening-orthorectification/README.md @@ -6,15 +6,15 @@ - In the associated `config.json`, there is a single directory listed. Xylem will parse through this main directory and concurrently process all available subdirectories. ```json "input": - "/path/to/input/directory" + "/data/test-cases" ``` - Multiple directories may also be specified. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. ```json "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ] ``` - Define each module: @@ -90,5 +90,6 @@ For more thorough discussions on variables and overall structure, along with a l - Module P also updates the middle of the filenames from `P1BS` for panchromatic imagery and `M1BS` for multispectral imagery to `S3XS` which is a Maxar naming standard and a naming convention upheld throughout the use of Legacy PIPE. - For multi-module processing, combinations can **only** be made in these two orders: > Module A -> Module P -> Module O + > Module P -> Module O - Atmospheric correction is **always** the first processing step if it is included, and imagery **must** be pansharpened before it is orthorectified. \ No newline at end of file diff --git a/examples/pansharpening-orthorectification/config.json b/examples/pansharpening-orthorectification/config.json index c5e202e..2ee5c58 100644 --- a/examples/pansharpening-orthorectification/config.json +++ b/examples/pansharpening-orthorectification/config.json @@ -7,7 +7,7 @@ ], "keywords": [], "input": - "/path/to/input/directory", + "/data/test-cases", "modules": [ { "name": "Module P (local)", @@ -15,7 +15,7 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-p", "variables": { - "OUTPUT_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/data/output", "METHOD": "nn_diffuse", "MODULE_LIST": "MODP, MODO" }, @@ -45,8 +45,8 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-o", "variables": { - "SOURCE_DIRECTORY": "/path/to/output/directory", - "OUTPUT_DIRECTORY": "/path/to/output/directory" + "SOURCE_DIRECTORY": "/data/output", + "OUTPUT_DIRECTORY": "/data/output" }, "template": { "command": "python", diff --git a/examples/pansharpening/README.md b/examples/pansharpening/README.md index 11ca52d..7301a65 100644 --- a/examples/pansharpening/README.md +++ b/examples/pansharpening/README.md @@ -6,16 +6,16 @@ - In the associated `config.json`, multiple directories are listed. Note that when listing multiple directories, they are listed within `[ ]`, signaling to Xylem these specific directories should be concurrently processed. ```json "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ] ``` - A single directory may also be specified. Xylem will parse through this main directory and concurrently process all available subdirectories. ```json "input": - "/path/to/input/directory" + "/data/test-cases" ``` - Define the module: - Within the module variable, specify the `name` and `uri` to the module. The `uri` specifically informs Xylem from where to install the module environment. For example, if a user has Xylem installed and has cloned each of the modules in their `dev` directory within a Docker container, this path may look something like: diff --git a/examples/pansharpening/config.json b/examples/pansharpening/config.json index 4fb965f..e093ebc 100644 --- a/examples/pansharpening/config.json +++ b/examples/pansharpening/config.json @@ -6,10 +6,10 @@ "conda" ], "input": [ - "/path/to/input/order1", - "/path/to/input/order2", - "/path/to/input/order3", - "/path/to/input/order4" + "/data/test-cases/test-case-GE01-medium-low-mid", + "/data/test-cases/test-case-WV02-medium-mid-high", + "/data/test-cases/test-case-wv03-large-high-mid", + "/data/test-cases/test-case-wv03-large-mid-low" ], "modules": [ { @@ -18,7 +18,7 @@ "programmingLanguage": "python", "uri": "file:///path/to/module-p", "variables": { - "OUTPUT_DIRECTORY": "/path/to/output/directory", + "OUTPUT_DIRECTORY": "/data/output", "METHOD": "nn_diffuse", "MODULE_LIST": "MODP" }, -- GitLab