Commit df10c905 authored by syz's avatar syz Committed by CompPhysChris
Browse files

Fixed bug with memory and core calc. Added noise threshold to grp attrs....

Fixed bug with memory and core calc. Added noise threshold to grp attrs. Always printing progress when writing.
parent a1fd4830
......@@ -85,38 +85,23 @@ class SignalFilter(Process):
self.write_filtered = write_filtered
self.write_condensed = write_condensed
self.data = None
self.filtered_data = None
self.condensed_data = None
self.noise_floors = None
self.h5_filtered = None
self.h5_condensed = None
self.h5_noise_floors = None
def _set_memory_and_cores(self, cores=1, mem=1024):
"""
Checks hardware limitations such as memory, # cpus and sets the recommended datachunk sizes and the
number of cores to be used by analysis methods.
Parameters
----------
cores : uint, optional
Default - 1
How many cores to use for the computation
mem : uint, optional
Default - 1024
The amount a memory in Mb to use in the computation
"""
super(SignalFilter, self)._set_memory_and_cores(cores=cores, mem=mem)
"""
Remember that the default number of pixels corresponds to only the raw data that can be held in memory
In the case of signal filtering, the datasets that will occupy space are:
1. Raw, 2. filtered (real + freq space copies), 3. Condensed (substantially lesser space)
The actual scaling of memory depends on options:
"""
scaling_factor = 1 + 2*self.write_filtered + 0.25*self.write_condensed
scaling_factor = 1 + 2 * self.write_filtered + 0.25 * self.write_condensed
self._max_pos_per_read = int(self._max_pos_per_read / scaling_factor)
self.data = None
self.filtered_data = None
self.condensed_data = None
self.noise_floors = None
self.h5_filtered = None
self.h5_condensed = None
self.h5_noise_floors = None
def _create_results_datasets(self):
"""
Creates all the datasets necessary for holding all parameters + data.
......@@ -124,12 +109,16 @@ class SignalFilter(Process):
grp_name = self.h5_main.name.split('/')[-1] + '-FFT_Filtering_'
grp_filt = MicroDataGroup(grp_name, self.h5_main.parent.name)
filter_parms = dict()
if self.frequency_filters is not None:
for filter in self.frequency_filters:
filter_parms.update(filter.get_parms())
if self.noise_threshold is not None:
filter_parms['noise_threshold'] = self.noise_threshold
filter_parms['algorithm'] = 'pycroscopy_SignalFilter'
filter_parms['last_pixel'] = 0
grp_filt.attrs = filter_parms
if isinstance(self.composite_filter, np.ndarray):
......@@ -241,8 +230,7 @@ class SignalFilter(Process):
self.hdf.flush()
if self.verbose:
print('Finished processing upto pixel ' + str(self._end_pos) + ' of ' + str(self.h5_main.shape[0]))
print('Finished processing upto pixel ' + str(self._end_pos) + ' of ' + str(self.h5_main.shape[0]))
# Now update the start position
self._start_pos = self._end_pos
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment