Commit fc683160 authored by Somnath, Suhas's avatar Somnath, Suhas
Browse files

Gmode-IV translator now does not use HDFWriter + VirtualData!

parent f70d4ea7
......@@ -332,7 +332,7 @@ class BEodfTranslator(Translator):
spec_label=spec_label, show_plots=show_plots, save_plots=save_plots,
do_histogram=do_histogram, debug=verbose)
self.h5_raw.file.close()
h5_f.close()
return h5_path
......
......@@ -14,10 +14,8 @@ import h5py
import numpy as np # For array operations
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import build_ind_val_dsets, Dimension
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter # Now the translator is responsible for writing the data.
from ...core.io.virtual_data import VirtualGroup, VirtualDataset # building blocks for defining heirarchical storage in the H5 file
from ...core.io.write_utils import Dimension
from ...core.io.hdf_utils import write_main_dataset, create_indexed_group, write_simple_attrs
class GIVTranslator(Translator):
......@@ -45,69 +43,52 @@ class GIVTranslator(Translator):
h5_path : string / unicode
Absolute path of the translated h5 file
"""
parm_path = path.abspath(parm_path)
parm_dict, excit_wfm = self._read_parms(parm_path)
folder_path, base_name = path.split(parm_path)
waste, base_name = path.split(folder_path)
# Until a better method is provided....
with h5py.File(path.join(folder_path, 'line_1.mat'), 'r') as h5_mat_line_1:
num_ai_chans = h5_mat_line_1['data'].shape[1]
h5_path = path.join(folder_path, base_name+'.h5')
if path.exists(h5_path):
remove(h5_path)
# Now start creating datasets and populating:
spec_desc = Dimension('Bias', 'V', excit_wfm)
ds_spec_inds, ds_spec_vals = build_ind_val_dsets(spec_desc, is_spectral=True, verbose=False)
ds_spec_vals.data = np.atleast_2d(excit_wfm) # The data generated above varies linearly. Override.
with h5py.File(h5_path) as h5_f:
pos_desc = Dimension('Y', 'm', np.linspace(0, parm_dict['grid_scan_height_[m]'], parm_dict['grid_num_rows']))
ds_pos_ind, ds_pos_val = build_ind_val_dsets(pos_desc, is_spectral=False, verbose=False)
# Minimize file size to the extent possible.
# DAQs are rated at 16 bit so float16 should be most appropriate.
# For some reason, compression is effective only on time series data
ds_raw_data = VirtualDataset('Raw_Data', data=None,
maxshape=(parm_dict['grid_num_rows'], excit_wfm.size),
dtype=np.float16, chunking=(1, excit_wfm.size), compression='gzip')
ds_raw_data.attrs['quantity'] = ['Current']
ds_raw_data.attrs['units'] = ['1E-{} A'.format(parm_dict['IO_amplifier_gain'])]
aux_ds_names = ['Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values']
# Until a better method is provided....
h5_f = h5py.File(path.join(folder_path, 'line_1.mat'), 'r')
num_ai_chans = h5_f['data'].shape[1]
h5_f.close()
# technically should change the date, etc.
spm_data = VirtualGroup('')
global_parms = generate_dummy_main_parms()
global_parms['data_type'] = 'gIV'
global_parms['translator'] = 'gIV'
spm_data.attrs = global_parms
meas_grp = VirtualGroup('Measurement_000')
spm_data.add_children([meas_grp])
hdf = HDFwriter(h5_path)
# spm_data.showTree()
hdf.write(spm_data, print_log=False)
h5_meas_grp = create_indexed_group(h5_f, 'Measurement')
global_parms = generate_dummy_main_parms()
global_parms.update({'data_type': 'gIV', 'translator': 'gIV'})
write_simple_attrs(h5_meas_grp, global_parms)
self.raw_datasets = list()
for chan_index in range(num_ai_chans):
chan_grp = VirtualGroup('{:s}{:03d}'.format('Channel_', chan_index), '/Measurement_000/')
chan_grp.attrs = parm_dict
chan_grp.add_children([ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals,
ds_raw_data])
h5_refs = hdf.write(chan_grp, print_log=False)
h5_raw = get_h5_obj_refs(['Raw_Data'], h5_refs)[0]
link_h5_objects_as_attrs(h5_raw, get_h5_obj_refs(aux_ds_names, h5_refs))
self.raw_datasets.append(h5_raw)
# Now that the N channels have been made, populate them with the actual data....
self._read_data(parm_dict, folder_path)
# Only prepare the instructions for the dimensions here
spec_dims = Dimension('Bias', 'V', excit_wfm)
pos_dims = Dimension('Y', 'm', np.linspace(0, parm_dict['grid_scan_height_[m]'],
parm_dict['grid_num_rows']))
self.raw_datasets = list()
for chan_index in range(num_ai_chans):
h5_chan_grp = create_indexed_group(h5_meas_grp, 'Channel')
write_simple_attrs(h5_chan_grp, parm_dict)
"""
Minimize file size to the extent possible.
DAQs are rated at 16 bit so float16 should be most appropriate.
For some reason, compression is effective only on time series data
"""
h5_raw = write_main_dataset(h5_chan_grp, (parm_dict['grid_num_rows'], excit_wfm.size), 'Raw_Data',
'Current',
'1E-{} A'.format(parm_dict['IO_amplifier_gain']), pos_dims, spec_dims,
dtype=np.float16, chunks=(1, excit_wfm.size), compression='gzip')
self.raw_datasets.append(h5_raw)
# Now that the N channels have been made, populate them with the actual data....
self._read_data(parm_dict, folder_path)
hdf.close()
return h5_path
def _read_data(self, parm_dict, folder_path):
......@@ -131,14 +112,15 @@ class GIVTranslator(Translator):
print('Reading data in line {} of {}'.format(line_ind+1, parm_dict['grid_num_rows']))
file_path = path.join(folder_path, 'line_'+str(line_ind+1)+'.mat')
if path.exists(file_path):
h5_f = h5py.File(file_path, 'r')
h5_data = h5_f['data']
if h5_data.shape[0] >= parm_dict['excitation_length'] and h5_data.shape[1] == len(self.raw_datasets):
for chan, h5_chan in enumerate(self.raw_datasets):
h5_chan[line_ind, :] = np.float16(h5_data[main_data, chan])
h5_chan.file.flush()
else:
warn('No data found for Line '+str(line_ind))
with h5py.File(file_path, 'r') as h5_f:
h5_data = h5_f['data']
if h5_data.shape[0] >= parm_dict['excitation_length'] and \
h5_data.shape[1] == len(self.raw_datasets):
for chan, h5_chan in enumerate(self.raw_datasets):
h5_chan[line_ind, :] = np.float16(h5_data[main_data, chan])
h5_chan.file.flush()
else:
warn('No data found for Line '+str(line_ind))
else:
warn('File not found for: line '+str(line_ind))
print('Finished reading all data!')
......@@ -160,56 +142,55 @@ class GIVTranslator(Translator):
excit_wfm : 1d numpy float array
Excitation waveform
"""
h5_f = h5py.File(parm_path, 'r')
parm_dict = dict()
parm_dict['IO_samp_rate_[Hz]'] = np.uint32(h5_f['samp_rate'][0][0])
parm_dict['IO_amplifier_gain'] = np.uint32(h5_f['amp_gain'][0][0])
with h5py.File(parm_path, 'r') as h5_f:
parm_dict = dict()
parm_dict['IO_samp_rate_[Hz]'] = np.uint32(h5_f['samp_rate'][0][0])
parm_dict['IO_amplifier_gain'] = np.uint32(h5_f['amp_gain'][0][0])
parm_dict['excitation_frequency_[Hz]'] = np.float32(h5_f['frequency'][0][0])
parm_dict['excitation_amplitude_[V]'] = np.float32(h5_f['amplitude'][0][0])
parm_dict['excitation_offset_[V]'] = np.float32(h5_f['offset'][0][0])
excit_wfm = np.float32(np.squeeze(h5_f['excit_wfm'].value))
# Make sure to truncate the data to the point when the
pts_per_cycle = int(np.round(1.0*parm_dict['IO_samp_rate_[Hz]']/parm_dict['excitation_frequency_[Hz]']))
extra_pts = len(excit_wfm) % pts_per_cycle
parm_dict['excitation_extra_pts'] = extra_pts
""" New versions could have used a pulse at the beginning of the line to "wake up" the material.
This pulse is always an integer number of cycles long
For now, let's remove this section from the excitation waveform and data"""
# This pulse may or may not have been used:
try:
pulse_duration = np.float32(h5_f['pulse_duration'][0][0])
pulse_height = np.float32(h5_f['pulse_height'][0][0])
except KeyError:
pulse_duration = 0.0
pulse_height = 0.0
if pulse_duration == 0.0:
pulse_height = 0.0
parm_dict['excitation_pulse_height_[V]'] = np.float32(pulse_height)
parm_dict['excitation_pulse_time_[sec]'] = np.float32(pulse_duration)
pulse_points = int(np.round(pulse_duration * parm_dict['IO_samp_rate_[Hz]']))
parm_dict['excitation_pulse_points'] = np.uint32(pulse_points)
line_time = np.float32(h5_f['line_time'][0][0]) - pulse_duration
excess_time = line_time - 1.0*extra_pts/parm_dict['IO_samp_rate_[Hz]']
parm_dict['excitation_duration_[sec]'] = line_time - excess_time
if extra_pts > 0:
excit_wfm = excit_wfm[pulse_points:-extra_pts]
else:
excit_wfm = excit_wfm[pulse_points:]
parm_dict['excitation_length'] = len(excit_wfm)
parm_dict['excitation_frequency_[Hz]'] = np.float32(h5_f['frequency'][0][0])
parm_dict['excitation_amplitude_[V]'] = np.float32(h5_f['amplitude'][0][0])
parm_dict['excitation_offset_[V]'] = np.float32(h5_f['offset'][0][0])
excit_wfm = np.float32(np.squeeze(h5_f['excit_wfm'].value))
# Make sure to truncate the data to the point when the
pts_per_cycle = int(np.round(1.0*parm_dict['IO_samp_rate_[Hz]']/parm_dict['excitation_frequency_[Hz]']))
extra_pts = len(excit_wfm) % pts_per_cycle
parm_dict['excitation_extra_pts'] = extra_pts
""" New versions could have used a pulse at the beginning of the line to "wake up" the material.
This pulse is always an integer number of cycles long
For now, let's remove this section from the excitation waveform and data"""
# This pulse may or may not have been used:
try:
pulse_duration = np.float32(h5_f['pulse_duration'][0][0])
pulse_height = np.float32(h5_f['pulse_height'][0][0])
except KeyError:
pulse_duration = 0.0
pulse_height = 0.0
if pulse_duration == 0.0:
pulse_height = 0.0
parm_dict['excitation_pulse_height_[V]'] = np.float32(pulse_height)
parm_dict['excitation_pulse_time_[sec]'] = np.float32(pulse_duration)
pulse_points = int(np.round(pulse_duration * parm_dict['IO_samp_rate_[Hz]']))
parm_dict['excitation_pulse_points'] = np.uint32(pulse_points)
line_time = np.float32(h5_f['line_time'][0][0]) - pulse_duration
excess_time = line_time - 1.0*extra_pts/parm_dict['IO_samp_rate_[Hz]']
parm_dict['excitation_duration_[sec]'] = line_time - excess_time
if extra_pts > 0:
excit_wfm = excit_wfm[pulse_points:-extra_pts]
else:
excit_wfm = excit_wfm[pulse_points:]
parm_dict['excitation_length'] = len(excit_wfm)
parm_dict['grid_num_rows'] = np.uint32(h5_f['num_lines'][0][0])
parm_dict['grid_num_cols'] = np.uint32(np.floor(len(excit_wfm) / pts_per_cycle))
parm_dict['grid_num_rows'] = np.uint32(h5_f['num_lines'][0][0])
parm_dict['grid_num_cols'] = np.uint32(np.floor(len(excit_wfm) / pts_per_cycle))
parm_dict['grid_scan_height_[m]'] = np.float32(h5_f['scan_height'][0][0])
parm_dict['grid_scan_width_[m]'] = np.float32(h5_f['scan_width'][0][0])
parm_dict['grid_scan_speed_[ms-1]'] = np.float32(h5_f['scan_speed'][0][0])
parm_dict['grid_scan_height_[m]'] = np.float32(h5_f['scan_height'][0][0])
parm_dict['grid_scan_width_[m]'] = np.float32(h5_f['scan_width'][0][0])
parm_dict['grid_scan_speed_[ms-1]'] = np.float32(h5_f['scan_speed'][0][0])
h5_f.close()
return parm_dict, excit_wfm
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment