Commit 8d7304d7 authored by syz's avatar syz
Browse files

First working version of SignalFilter! Outputs not tested yet

parent e1751b57
......@@ -13,15 +13,15 @@ import numpy as np
from collections import Iterable
from .process import Process, parallel_compute
from ..io.microdata import MicroDataset, MicroDataGroup
from ..io.hdf_utils import getH5DsetRefs, getAuxData, copyAttributes, link_as_main, linkRefs
from ..io.hdf_utils import getH5DsetRefs, getAuxData, copyAttributes, link_as_main, linkRefs, print_tree
from ..io.translators.utils import build_ind_val_dsets
from ..io.io_hdf5 import ioHDF5
from .fft import getNoiseFloor, are_compatible_filters, build_composite_freq_filter
class SignalFilter(Process):
def __init__(self, h5_main, frequency_filters=None, noise_threshold=None, write_filtered=True, write_condensed=False,
def __init__(self, h5_main, frequency_filters=None, noise_threshold=None, write_filtered=True,
write_condensed=False,
num_pix=1, phase_rad=0, verbose=False, cores=None, max_mem_mb=1024, **kwargs):
super(SignalFilter, self).__init__(h5_main, cores=cores, max_mem_mb=max_mem_mb, verbose=verbose, **kwargs)
......@@ -59,7 +59,6 @@ class SignalFilter(Process):
self.write_filtered = write_filtered
self.write_condensed = write_condensed
def _create_results_datasets(self):
"""
Process specific call that will write the h5 group, guess dataset, corresponding spectroscopic datasets and also
......@@ -74,14 +73,22 @@ class SignalFilter(Process):
for filter in self.frequency_filters:
filter_parms.update(filter.get_parms())
filter_parms['algorithm'] = 'GmodeUtils-Parallel'
filter_parms['last_pixel'] = 0
grp_filt.attrs = filter_parms
if isinstance(self.composite_filter, np.ndarray):
ds_comp_filt = MicroDataset('Composite_Filter', np.float32(self.composite_filter))
grp_filt.addChildren([ds_comp_filt])
if self.noise_threshold is not None:
ds_noise_floors = MicroDataset('Noise_Floors',
data=np.zeros(shape=self.num_effective_pix, dtype=np.float32))
grp_filt.addChildren([ds_noise_floors])
data=np.zeros(shape=(self.num_effective_pix, 1), dtype=np.float32))
ds_noise_spec_inds, ds_noise_spec_vals = build_ind_val_dsets([1], is_spectral=True,
labels=['arb'], units=[''],
verbose=self.verbose)
ds_noise_spec_inds.name = 'Noise_Spectral_Indices'
ds_noise_spec_vals.name = 'Noise_Spectral_Values'
grp_filt.addChildren([ds_noise_floors, ds_noise_spec_inds, ds_noise_spec_vals])
if self.write_filtered:
ds_filt_data = MicroDataset('Filtered_Data', data=[], maxshape=self.h5_main.maxshape,
......@@ -97,9 +104,11 @@ class SignalFilter(Process):
self.hot_inds = np.where(self.composite_filter > 0)[0]
self.hot_inds = np.uint(self.hot_inds[int(0.5 * len(self.hot_inds)):]) # only need to keep half the data
ds_spec_inds, ds_spec_vals = build_ind_val_dsets([int(0.5 * len(self.hot_inds))], is_spectral=True,
labels=['hot_frequencies'], units=[''], verbose=False)
labels=['hot_frequencies'], units=[''],
verbose=self.verbose)
ds_spec_vals.data = np.atleast_2d(self.hot_inds) # The data generated above varies linearly. Override.
ds_cond_data = MicroDataset('Condensed_Data', data=[], maxshape=(self.num_effective_pix, len(self.hot_inds)),
ds_cond_data = MicroDataset('Condensed_Data', data=[],
maxshape=(self.num_effective_pix, len(self.hot_inds)),
dtype=np.complex, chunking=(1, len(self.hot_inds)), compression='gzip')
grp_filt.addChildren([ds_spec_inds, ds_spec_vals, ds_cond_data])
if self.num_effective_pix > 1:
......@@ -110,17 +119,24 @@ class SignalFilter(Process):
for dim_ind in range(h5_pos_inds.shape[1])],
is_spectral=False,
labels=h5_pos_inds.attrs['labels'],
units=h5_pos_inds.attrs['units'], verbose=False)
units=h5_pos_inds.attrs['units'], verbose=self.verbose)
h5_pos_vals.data = np.atleast_2d(new_pos_vals) # The data generated above varies linearly. Override.
grp_filt.addChildren([ds_pos_inds, ds_pos_vals])
if self.verbose:
grp_filt.showTree()
hdf = ioHDF5(self.h5_main.file)
h5_filt_refs = hdf.writeData(grp_filt)
h5_filt_refs = hdf.writeData(grp_filt, print_log=self.verbose)
if isinstance(self.composite_filter, np.ndarray):
h5_comp_filt = getH5DsetRefs(['Composite_Filter'], h5_filt_refs)[0]
if self.noise_threshold is not None:
self.h5_noise_floors = getH5DsetRefs(['Noise_Floors'], h5_filt_refs)[0]
self.h5_results_grp = self.h5_noise_floors.parent
link_as_main(self.h5_noise_floors, h5_pos_inds, h5_pos_vals,
getH5DsetRefs(['Noise_Spectral_Indices'], h5_filt_refs)[0],
getH5DsetRefs(['Noise_Spectral_Values'], h5_filt_refs)[0])
# Now need to link appropriately:
if self.write_filtered:
......@@ -129,8 +145,6 @@ class SignalFilter(Process):
copyAttributes(self.h5_main, self.h5_filtered, skip_refs=False)
if isinstance(self.composite_filter, np.ndarray):
linkRefs(self.h5_filtered, [h5_comp_filt])
if self.noise_threshold is not None:
linkRefs(self.h5_filtered, [self.h5_noise_floors])
"""link_as_main(self.h5_filtered, h5_pos_inds, h5_pos_vals,
getAuxData(h5_main, auxDataName=['Spectroscopic_Indices'])[0],
......@@ -159,15 +173,18 @@ class SignalFilter(Process):
if self.write_condensed:
self.h5_condensed[pos_slice] = self.condensed_data
if self.noise_threshold is not None:
self.h5_noise_floors[pos_slice] = self.noise_floors
self.h5_noise_floors[pos_slice] = np.atleast_2d(self.noise_floors)
if self.write_filtered:
self.h5_filtered[pos_slice] = self.filtered_data
# Leaving in this provision that will allow restarting of processes
self.h5_results_grp['last_pixel'] = self._end_pos
self.h5_results_grp.attrs['last_pixel'] = self._end_pos
self.hdf.flush()
if self.verbose:
print('Finished processing upto pixel ' + str(self._end_pos) + ' of ' + str(self.h5_main.shape[0]))
# Now update the start position
self._start_pos = self._end_pos
......@@ -212,7 +229,7 @@ class SignalFilter(Process):
if self.noise_threshold is not None:
# apply thresholding
self.data[self.data < np.tile(np.atleast_2d(self.noise_floors).T, self.noise_floors.shape[1])] = 1E-16
self.data[self.data < np.tile(np.atleast_2d(self.noise_floors), self.data.shape[1])] = 1E-16
pass
if self.write_condensed:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment