Commit 9eb80499 authored by Zhang, Chen's avatar Zhang, Chen
Browse files

explicit specify chunksize for multiprocess

parent 6f2419d2

src/imars3d/backend/corrections/beam_hardening.py

+2 −1
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ 
import logging

import param

import numpy as np

from imars3d.backend.util.functions import clamp_max_workers

from imars3d.backend.util.functions import clamp_max_workers, calculate_chunksize

from multiprocessing.managers import SharedMemoryManager

from functools import partial

from tqdm.contrib.concurrent import process_map
@@ -83,6 +83,7 @@ class beam_hardening_correction(param.ParameterizedFunction): 
                # mp

                kwargs = {

                    "max_workers": self.max_workers,

                    "chunksize": calculate_chunksize(params.arrays.shape[0], self.max_workers),

                    "desc": "denoise_by_bilateral",

                }

                if self.tqdm_class:

src/imars3d/backend/corrections/denoise.py

+2 −1
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
"""Image noise reduction (denoise) module."""

import logging

import param

from imars3d.backend.util.functions import clamp_max_workers

from imars3d.backend.util.functions import clamp_max_workers, calculate_chunksize

import numpy as np

import tomopy

from multiprocessing.managers import SharedMemoryManager
@@ -153,6 +153,7 @@ def denoise_by_bilateral( 
            # mp

            kwargs = {

                "max_workers": max_workers,

                "chunksize": calculate_chunksize(arrays.shape[0], max_workers),

                "desc": "denoise_by_bilateral",

            }

            if tqdm_class:

src/imars3d/backend/corrections/intensity_fluctuation_correction.py

+2 −1
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
# -*- coding: utf-8 -*-

"""iMars3D's intensity fluctuation correction module."""

import logging

from imars3d.backend.util.functions import clamp_max_workers

from imars3d.backend.util.functions import clamp_max_workers, calculate_chunksize

import numpy as np

import param

import tomopy
@@ -93,6 +93,7 @@ class intensity_fluctuation_correction(param.ParameterizedFunction): 
                # map the multiprocessing calls

                kwargs = {

                    "max_workers": max_workers,

                    "chunksize": calculate_chunksize(ct.shape[0], max_workers),

                    "desc": "intensity_fluctuation_correction",

                }

                if tqdm_class:

src/imars3d/backend/corrections/ring_removal.py

+2 −1
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
"""iMars3D's ring artifact correction module."""

import logging

import param

from imars3d.backend.util.functions import clamp_max_workers

from imars3d.backend.util.functions import clamp_max_workers, calculate_chunksize

import scipy

import numpy as np
@@ -238,6 +238,7 @@ class remove_ring_artifact(param.ParameterizedFunction): 
            # invoke mp via tqdm wrapper

            kwargs = {

                "max_workers": max_workers,

                "chunksize": calculate_chunksize(arrays.shape[1], max_workers),

                "desc": "Removing ring artifact",

            }

            if tqdm_class:

src/imars3d/backend/dataio/data.py

+2 −1
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 


# package imports

from imars3d.backend.dataio.metadata import MetaData

from imars3d.backend.util.functions import clamp_max_workers, to_time_str

from imars3d.backend.util.functions import clamp_max_workers, to_time_str, calculate_chunksize



# third party imports

import numpy as np
@@ -329,6 +329,7 @@ def _load_images(filelist: List[str], desc: str, max_workers: int, tqdm_class) - 
        #       - there are a lot of cores available

        kwargs = {

            "max_workers": max_workers,

            "chunksize": calculate_chunksize(len(filelist), max_workers),

            "desc": desc,

        }

        rst = process_map(partial(_forgiving_reader, reader=reader), filelist, **kwargs)