Commit 1291740a authored by Somnath, Suhas's avatar Somnath, Suhas Committed by Chris Smith
Browse files

Using the centralized dtypes for indices and values datasets everywhere

parent aff4187a
......@@ -13,7 +13,7 @@ from warnings import warn
from collections import Iterable
import numpy as np
from .virtual_data import VirtualDataset
from .write_utils import make_indices_matrix
from .write_utils import make_indices_matrix, INDICES_DTYPE, VALUES_DTYPE
__all__ = ['get_attr', 'get_h5_obj_refs', 'get_indices_for_region_ref', 'get_dimensionality', 'get_sort_order',
'get_auxillary_datasets', 'get_attributes', 'get_group_refs', 'check_if_main', 'check_and_link_ancillary',
......@@ -722,12 +722,12 @@ def reshape_to_n_dims(h5_main, h5_pos=None, h5_spec=None, get_labels=False, verb
except KeyError:
print('No position datasets found as attributes of {}'.format(h5_main.name))
if len(h5_main.shape) > 1:
ds_pos = np.arange(h5_main.shape[0], dtype=np.uint8).reshape(-1, 1)
ds_pos = np.arange(h5_main.shape[0], dtype=INDICES_DTYPE).reshape(-1, 1)
pos_labs = np.array(['Position Dimension {}'.format(ipos) for ipos in range(ds_pos.shape[1])])
else:
ds_pos = np.array(0, dtype=np.uint8).reshape(-1, 1)
ds_pos = np.array(0, dtype=INDICES_DTYPE).reshape(-1, 1)
else:
ds_pos = np.arange(h5_main.shape[0], dtype=np.uint32).reshape(-1, 1)
ds_pos = np.arange(h5_main.shape[0], dtype=INDICES_DTYPE).reshape(-1, 1)
pos_labs = np.array(['Position Dimension {}'.format(ipos) for ipos in range(ds_pos.shape[1])])
elif isinstance(h5_pos, h5py.Dataset):
"""
......@@ -755,12 +755,12 @@ def reshape_to_n_dims(h5_main, h5_pos=None, h5_spec=None, get_labels=False, verb
except KeyError:
print('No spectroscopic datasets found as attributes of {}'.format(h5_main.name))
if len(h5_main.shape) > 1:
ds_spec = np.arange(h5_main.shape[1], dtype=np.uint8).reshape([1, -1])
ds_spec = np.arange(h5_main.shape[1], dtype=INDICES_DTYPE).reshape([1, -1])
spec_labs = np.array(['Spectral Dimension {}'.format(ispec) for ispec in range(ds_spec.shape[0])])
else:
ds_spec = np.array(0, dtype=np.uint8).reshape([1, 1])
ds_spec = np.array(0, dtype=INDICES_DTYPE).reshape([1, 1])
else:
ds_spec = np.arange(h5_main.shape[1], dtype=np.uint8).reshape([1, -1])
ds_spec = np.arange(h5_main.shape[1], dtype=INDICES_DTYPE).reshape([1, -1])
spec_labs = np.array(['Spectral Dimension {}'.format(ispec) for ispec in range(ds_spec.shape[0])])
elif isinstance(h5_spec, h5py.Dataset):
......@@ -1546,8 +1546,8 @@ def build_reduced_spec_dsets(h5_spec_inds, h5_spec_vals, keep_dim, step_starts,
ds_vals.attrs['units'] = h5_spec_vals.attrs['units'][keep_dim]
else: # Single spectroscopic dimension:
ds_inds = VirtualDataset('Spectroscopic_Indices', np.array([[0]], dtype=np.uint32))
ds_vals = VirtualDataset('Spectroscopic_Values', np.array([[0]], dtype=np.float32))
ds_inds = VirtualDataset('Spectroscopic_Indices', np.array([[0]], dtype=INDICES_DTYPE))
ds_vals = VirtualDataset('Spectroscopic_Values', np.array([[0]], dtype=VALUES_DTYPE))
ds_inds.attrs['labels'] = {'Single_Step': (slice(0, None), slice(None))}
ds_vals.attrs['labels'] = {'Single_Step': (slice(0, None), slice(None))}
......
......@@ -9,7 +9,7 @@ from __future__ import division, print_function, absolute_import, unicode_litera
import numpy as np # For array operations
from .translator import Translator
from .write_utils import build_ind_val_dsets
from .write_utils import build_ind_val_dsets, VALUES_DTYPE
from .hdf_utils import calc_chunks
from .virtual_data import VirtualDataset # building blocks for defining hierarchical storage in the H5 file
......@@ -59,9 +59,9 @@ class NumpyTranslator(Translator):
spectra_length = main_data.shape[1]
ds_main = VirtualDataset('Raw_Data', data=main_data, dtype=np.float32, compression='gzip',
chunking=calc_chunks(main_data.shape, np.float32(0).itemsize,
unit_chunks=(1, spectra_length)))
ds_main = VirtualDataset('Raw_Data', data=main_data, dtype=VALUES_DTYPE, compression='gzip',
chunking=calc_chunks(main_data.shape, VALUES_DTYPE(0).itemsize,
unit_chunks=(1, spectra_length)))
ds_main.attrs = {'quantity': qty_name, 'units': data_unit}
pos_steps = None
......@@ -75,7 +75,7 @@ class NumpyTranslator(Translator):
labels=[spec_name], units=[spec_unit], verbose=False)
if spec_val is not None:
if type(spec_val) in [list, np.ndarray]:
ds_spec_vals.data = np.float32(np.atleast_2d(spec_val))
ds_spec_vals.data = VALUES_DTYPE(np.atleast_2d(spec_val))
parms_dict.update({'translator': 'NumpyTranslator'})
......
......@@ -5,7 +5,7 @@ from .virtual_data import VirtualDataset
from .dtype_utils import contains_integers
import warnings
__all__ = ['build_ind_val_dsets', 'get_aux_dset_slicing', 'make_indices_matrix']
__all__ = ['build_ind_val_dsets', 'get_aux_dset_slicing', 'make_indices_matrix', 'INDICES_DTYPE', 'VALUES_DTYPE']
if sys.version_info.major == 3:
unicode = str
......
......@@ -6,8 +6,8 @@ from . import gmode_iv
from . import gmode_line
from . import gmode_tune
from . import image
from . import ndata_translator
from . import trKPFM_translator
from . import ndata
from . import tr_kpfm
from . import igor_ibw
from . import oneview
from . import ptychography
......@@ -26,8 +26,8 @@ from .gmode_line import GLineTranslator
from .gmode_tune import GTuneTranslator
from .igor_ibw import IgorIBWTranslator
from .image import ImageTranslator
from .ndata_translator import NDataTranslator
from .trKPFM_translator import TRKPFMTranslator
from .ndata import NDataTranslator
from .tr_kpfm import TRKPFMTranslator
from .oneview import OneViewTranslator
from .ptychography import PtychographyTranslator
from .sporc import SporcTranslator
......
......@@ -16,7 +16,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
createSpecVals, requires_conjugate, nf32
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import build_ind_val_dsets
from ...core.io.write_utils import build_ind_val_dsets, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs, calc_chunks
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -169,7 +169,7 @@ class BEodfTranslator(Translator):
# Remove the unused plot group columns before proceeding:
(UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
spec_inds = np.zeros(shape=(2, tot_bins), dtype=np.uint)
spec_inds = np.zeros(shape=(2, tot_bins), dtype=INDICES_DTYPE)
# Will assume that all excitation waveforms have same number of bins
num_actual_udvs_steps = UDVS_mat.shape[0] / udvs_denom
......@@ -188,9 +188,9 @@ class BEodfTranslator(Translator):
if UDVS_mat[step_index, 2] < 1E-3: # invalid AC amplitude
continue
# Bin step
spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=np.uint32)
spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=INDICES_DTYPE)
# UDVS step
spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step, dtype=np.uint32)
spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step, dtype=INDICES_DTYPE)
stind += bins_per_step
del stind, step_index
......@@ -204,7 +204,7 @@ class BEodfTranslator(Translator):
UDVS_mat = np.array([1, 0, parm_dict['BE_amplitude_[V]'], 1, 1, 1],
dtype=np.float32).reshape(1, len(UDVS_labs))
spec_inds = np.vstack((np.arange(tot_bins, dtype=np.uint), np.zeros(tot_bins, dtype=np.uint32)))
spec_inds = np.vstack((np.arange(tot_bins, dtype=INDICES_DTYPE), np.zeros(tot_bins, dtype=INDICES_DTYPE)))
# Some very basic information that can help the processing / analysis crew
parm_dict['num_bins'] = tot_bins
......@@ -221,7 +221,7 @@ class BEodfTranslator(Translator):
ds_UDVS_inds = VirtualDataset('UDVS_Indices', spec_inds[1])
# ds_spec_labs = MicroDataset('Spectroscopic_Labels',np.array(['Bin','UDVS_Step']))
ds_bin_steps = VirtualDataset('Bin_Step', np.arange(bins_per_step, dtype=np.uint32), dtype=np.uint32)
ds_bin_steps = VirtualDataset('Bin_Step', np.arange(bins_per_step, dtype=INDICES_DTYPE), dtype=INDICES_DTYPE)
# Need to add the Bin Waveform type - infer from UDVS
exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
......@@ -234,7 +234,7 @@ class BEodfTranslator(Translator):
# This is wrong but I don't know what else to do
bin_FFT = np.hstack((bin_FFT, bin_FFT))
ds_bin_inds = VirtualDataset('Bin_Indices', bin_inds, dtype=np.uint32)
ds_bin_inds = VirtualDataset('Bin_Indices', bin_inds, dtype=INDICES_DTYPE)
ds_bin_freq = VirtualDataset('Bin_Frequencies', bin_freqs)
ds_bin_FFT = VirtualDataset('Bin_FFT', bin_FFT)
ds_wfm_typ = VirtualDataset('Bin_Wfm_Type', exec_bin_vec)
......@@ -256,10 +256,10 @@ class BEodfTranslator(Translator):
for row_ind, row_name in enumerate(spec_vals_labs):
spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=INDICES_DTYPE)
ds_spec_mat.attrs['labels'] = spec_vals_slices
ds_spec_mat.attrs['units'] = spec_vals_units
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=VALUES_DTYPE))
ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
ds_spec_vals_mat.attrs['units'] = spec_vals_units
for entry in spec_vals_labs_names:
......
......@@ -16,7 +16,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, generatePlotGroups, createSpecVals, maxReadPixels, \
nf32
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -127,9 +127,9 @@ class BEodfRelaxationTranslator(Translator):
pos_slices = get_aux_dset_slicing(['X', 'Y'], last_ind=num_pix, is_spectroscopic=False)
ds_ex_wfm = VirtualDataset('Excitation_Waveform', ex_wfm)
ds_pos_ind = VirtualDataset('Position_Indices', pos_mat, dtype=np.uint32)
ds_pos_ind = VirtualDataset('Position_Indices', pos_mat, dtype=INDICES_DTYPE)
ds_pos_ind.attrs['labels'] = pos_slices
ds_pos_val = VirtualDataset('Position_Values', np.float32(pos_mat))
ds_pos_val = VirtualDataset('Position_Values', VALUES_DTYPE(pos_mat))
ds_pos_val.attrs['labels'] = pos_slices
(UDVS_labs, UDVS_units, UDVS_mat) = self.__buildUDVSTable(parm_dict)
......@@ -137,7 +137,7 @@ class BEodfRelaxationTranslator(Translator):
# Remove the unused plot group columns before proceeding:
(UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
spec_inds = np.zeros(shape=(2, tot_bins), dtype=np.uint)
spec_inds = np.zeros(shape=(2, tot_bins), dtype=INDICES_DTYPE)
# Will assume that all excitation waveforms have same number of bins
# Here, the denominator is 2 because only out of field measruements. For IF + OF, should be 1
......@@ -157,9 +157,9 @@ class BEodfRelaxationTranslator(Translator):
for step_index in range(UDVS_mat.shape[0]):
if UDVS_mat[step_index, 2] < 1E-3: # invalid AC amplitude
continue # skip
spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=np.uint32) # Bin step
spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=INDICES_DTYPE) # Bin step
spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step,
dtype=np.uint32) # UDVS step
dtype=INDICES_DTYPE) # UDVS step
stind += bins_per_step
del stind, step_index
......@@ -175,7 +175,7 @@ class BEodfRelaxationTranslator(Translator):
ds_UDVS.attrs['labels'] = udvs_slices
ds_UDVS.attrs['units'] = UDVS_units
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=INDICES_DTYPE)
ds_spec_mat.attrs['labels'] = {'UDVS_Step': (slice(1, 2), slice(None)), 'Bin': (slice(0, 1), slice(None))}
ds_bin_steps = VirtualDataset('Bin_Step', np.arange(bins_per_step, dtype=np.uint32), dtype=np.uint32)
......@@ -197,7 +197,7 @@ class BEodfRelaxationTranslator(Translator):
spec_vals_slices = dict()
for row_ind, row_name in enumerate(spec_vals_labs):
spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=VALUES_DTYPE))
ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
ds_spec_vals_mat.attrs['units'] = spec_vals_units
......
......@@ -18,7 +18,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
normalizeBEresponse, createSpecVals, nf32
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import make_indices_matrix
from ...core.io.write_utils import make_indices_matrix, VALUES_DTYPE, INDICES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs, calc_chunks
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -282,7 +282,7 @@ class BEPSndfTranslator(Translator):
ds_pos_ind = VirtualDataset('Position_Indices',
self.pos_mat[self.ds_pixel_start_indx:self.ds_pixel_start_indx +
self.ds_pixel_index, :],
dtype=np.uint)
dtype=INDICES_DTYPE)
ds_pos_ind.attrs['labels'] = pos_slice_dict
ds_pos_ind.attrs['units'] = self.pos_units
......@@ -305,7 +305,7 @@ class BEPSndfTranslator(Translator):
# Z spectroscopy
self.pos_vals_list[:, 2] *= 1E+6 # convert to microns
pos_val_mat = np.float32(self.pos_mat[self.ds_pixel_start_indx:self.ds_pixel_start_indx +
pos_val_mat = VALUES_DTYPE(self.pos_mat[self.ds_pixel_start_indx:self.ds_pixel_start_indx +
self.ds_pixel_index, :])
for col_ind, targ_dim_name in enumerate(['X', 'Y', 'Z']):
......@@ -391,7 +391,7 @@ class BEPSndfTranslator(Translator):
del pixl, stind
# Make the index matrix that has the UDVS step number and bin indices
spec_inds = np.zeros(shape=(2, tot_pts), dtype=np.uint32)
spec_inds = np.zeros(shape=(2, tot_pts), dtype=INDICES_DTYPE)
stind = 0
# Need to go through the UDVS file and reconstruct chronologically
for step_index, wave_type in enumerate(self.excit_type_vec):
......@@ -457,10 +457,10 @@ class BEPSndfTranslator(Translator):
spec_vals_slices = dict()
for row_ind, row_name in enumerate(spec_vals_labs):
spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
ds_spec_vals_mat = VirtualDataset('Spectroscopic_Values', np.array(spec_vals, dtype=VALUES_DTYPE))
ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
ds_spec_vals_mat.attrs['units'] = spec_vals_units
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
ds_spec_mat = VirtualDataset('Spectroscopic_Indices', spec_inds, dtype=INDICES_DTYPE)
ds_spec_mat.attrs['labels'] = spec_vals_slices
ds_spec_mat.attrs['units'] = spec_vals_units
for entry in spec_vals_labs_names:
......
......@@ -15,7 +15,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.gmode_utils import readGmodeParms
from ...core.io.translator import Translator, \
generate_dummy_main_parms # Because this class extends the abstract Translator class
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter # Now the translator is responsible for writing the data.
# The building blocks for defining heirarchical storage in the H5 file
......@@ -69,7 +69,7 @@ class GDMTranslator(Translator):
w_vec = np.arange(-0.5 * samp_rate, 0.5 * samp_rate, np.float32(samp_rate / num_bins))
# There is most likely a more elegant solution to this but I don't have the time... Maybe np.meshgrid
spec_val_mat = np.zeros((len(freq_array) * num_bins, 2), dtype=np.float32)
spec_val_mat = np.zeros((len(freq_array) * num_bins, 2), dtype=VALUES_DTYPE)
spec_val_mat[:, 0] = np.tile(w_vec, len(freq_array))
spec_val_mat[:, 1] = np.repeat(freq_array, num_bins)
......@@ -87,12 +87,12 @@ class GDMTranslator(Translator):
pos_slices = get_aux_dset_slicing(['X', 'Y'], last_ind=num_pix, is_spectroscopic=False)
# Now start creating datasets and populating:
ds_pos_ind = VirtualDataset('Position_Indices', np.uint32(pos_mat))
ds_pos_ind = VirtualDataset('Position_Indices', INDICES_DTYPE(pos_mat))
ds_pos_ind.attrs['labels'] = pos_slices
ds_pos_val = VirtualDataset('Position_Values', np.float32(pos_mat))
ds_pos_val = VirtualDataset('Position_Values', VALUES_DTYPE(pos_mat))
ds_pos_val.attrs['labels'] = pos_slices
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', np.uint32(spec_ind_mat))
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', INDICES_DTYPE(spec_ind_mat))
ds_spec_inds.attrs['labels'] = {'Response Bin Index': (slice(0, 1), slice(None)),
'Excitation Frequency Index': (slice(1, 2), slice(None))}
......
......@@ -14,7 +14,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.be_utils import parmsToDict
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import build_ind_val_dsets
from ...core.io.write_utils import build_ind_val_dsets, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -142,7 +142,7 @@ class GLineTranslator(Translator):
labels=['Y'], units=['m'])
ds_spec_inds, ds_spec_vals = build_ind_val_dsets([self.points_per_pixel * num_cols], is_spectral=True,
labels=['Excitation'], units=['V'])
ds_spec_vals.data = np.atleast_2d(np.tile(np.float32(be_wave), num_cols)) # Override the default waveform
ds_spec_vals.data = np.atleast_2d(np.tile(VALUES_DTYPE(be_wave), num_cols)) # Override the default waveform
aux_ds_names = ['Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values']
......
......@@ -17,7 +17,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from .df_utils.be_utils import parmsToDict
from .gmode_line import GLineTranslator
from ...core.io.translator import generate_dummy_main_parms
from ...core.io.write_utils import build_ind_val_dsets
from ...core.io.write_utils import build_ind_val_dsets, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -152,7 +152,7 @@ class GTuneTranslator(GLineTranslator):
labels=['Y'], units=['m'])
ds_spec_inds, ds_spec_vals = build_ind_val_dsets([self.points_per_pixel * num_cols], is_spectral=True,
labels=['Excitation'], units=['V'])
ds_spec_vals.data = np.atleast_2d(np.tile(np.float32(be_wave), num_cols)) # Override the default waveform
ds_spec_vals.data = np.atleast_2d(np.tile(VALUES_DTYPE(be_wave), num_cols)) # Override the default waveform
aux_ds_names = ['Position_Indices', 'Position_Values',
'Spectroscopic_Indices', 'Spectroscopic_Values']
......
......@@ -13,7 +13,7 @@ from igor import binarywave as bw
from ...core.io.translator import Translator, \
generate_dummy_main_parms # Because this class extends the abstract Translator class
from ...core.io.write_utils import build_ind_val_dsets
from ...core.io.write_utils import build_ind_val_dsets, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter # Now the translator is responsible for writing the data.
from ...core.io.virtual_data import VirtualGroup, \
......@@ -99,11 +99,11 @@ class IgorIBWTranslator(Translator):
# Find the channel that corresponds to either Z sensor or Raw:
try:
chan_ind = chan_labels.index('ZSnsr')
ds_spec_vals.data = np.atleast_2d(np.float32(images[chan_ind]))
ds_spec_vals.data = np.atleast_2d(VALUES_DTYPE(images[chan_ind]))
except ValueError:
try:
chan_ind = chan_labels.index('Raw')
ds_spec_vals.data = np.atleast_2d(np.float32(images[chan_ind]))
ds_spec_vals.data = np.atleast_2d(VALUES_DTYPE(images[chan_ind]))
except ValueError:
# We don't expect to come here. If we do, spectroscopic values remains as is
pass
......
......@@ -17,7 +17,7 @@ from skimage.util import crop
from .df_utils.io_image import unnest_parm_dicts, read_dm3
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, build_ind_val_dsets
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, build_ind_val_dsets, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, get_group_refs, calc_chunks, link_as_main
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -175,10 +175,10 @@ class NDataTranslator(Translator):
Build Spectroscopic and Position datasets for the image
'''
pos_mat = make_indices_matrix(image.shape)
spec_mat = np.array([[0]], dtype=np.uint8)
spec_mat = np.array([[0]], dtype=INDICES_DTYPE)
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', spec_mat)
ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_mat, dtype=np.float32)
ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_mat, dtype=VALUES_DTYPE)
spec_lab = get_aux_dset_slicing(['Image'], is_spectroscopic=True)
ds_spec_inds.attrs['labels'] = spec_lab
ds_spec_inds.attrs['units'] = ''
......@@ -186,7 +186,7 @@ class NDataTranslator(Translator):
ds_spec_vals.attrs['units'] = ''
ds_pos_inds = VirtualDataset('Position_Indices', pos_mat)
ds_pos_vals = VirtualDataset('Position_Values', pos_mat, dtype=np.float32)
ds_pos_vals = VirtualDataset('Position_Values', pos_mat, dtype=VALUES_DTYPE)
pos_lab = get_aux_dset_slicing(['X', 'Y'], is_spectroscopic=False)
ds_pos_inds.attrs['labels'] = pos_lab
......
......@@ -16,7 +16,7 @@ from skimage.util import crop
from .df_utils import dm4reader
from .df_utils.io_image import read_image, read_dm3, parse_dm4_parms
from ...core.io.translator import Translator, generate_dummy_main_parms
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, build_ind_val_dsets
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, build_ind_val_dsets, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, calc_chunks, link_as_main
from ...core.io.hdf_writer import HDFwriter
from ...core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -204,7 +204,7 @@ class OneViewTranslator(Translator):
spec_mat = np.array([[0]], dtype=np.uint8)
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', spec_mat)
ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_mat, dtype=np.float32)
ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_mat, dtype=VALUES_DTYPE)
spec_lab = get_aux_dset_slicing(['Image'], is_spectroscopic=True)
ds_spec_inds.attrs['labels'] = spec_lab
ds_spec_inds.attrs['units'] = ''
......@@ -212,7 +212,7 @@ class OneViewTranslator(Translator):
ds_spec_vals.attrs['units'] = ''
ds_pos_inds = VirtualDataset('Position_Indices', pos_mat)
ds_pos_vals = VirtualDataset('Position_Values', pos_mat, dtype=np.float32)
ds_pos_vals = VirtualDataset('Position_Values', pos_mat, dtype=VALUES_DTYPE)
pos_lab = get_aux_dset_slicing(['X', 'Y'], is_spectroscopic=False)
ds_pos_inds.attrs['labels'] = pos_lab
......
......@@ -15,7 +15,7 @@ from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat
from ...core.io.translator import Translator, \
generate_dummy_main_parms # Because this class extends the abstract Translator class
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing
from ...core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, INDICES_DTYPE, VALUES_DTYPE
from ...core.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
from ...core.io.hdf_writer import HDFwriter # Now the translator is responsible for writing the data.
# The building blocks for defining heirarchical storage in the H5 file
......@@ -65,12 +65,12 @@ class SporcTranslator(Translator):
pos_slices = get_aux_dset_slicing(['X', 'Y'], last_ind=num_pix, is_spectroscopic=False)
# new data format
spec_ind_mat = np.transpose(np.float32(spec_ind_mat))
spec_ind_mat = np.transpose(VALUES_DTYPE(spec_ind_mat))
# Now start creating datasets and populating:
ds_pos_ind = VirtualDataset('Position_Indices', np.uint32(pos_mat))
ds_pos_ind = VirtualDataset('Position_Indices', INDICES_DTYPE(pos_mat))
ds_pos_ind.attrs['labels'] = pos_slices
ds_pos_val = VirtualDataset('Position_Values', np.float32(pos_mat))
ds_pos_val = VirtualDataset('Position_Values', VALUES_DTYPE(pos_mat))
ds_pos_val.attrs['labels'] = pos_slices
ds_pos_val.attrs['units'] = ['um', 'um']
......@@ -78,7 +78,7 @@ class SporcTranslator(Translator):
spec_ind_dict = dict()
for col_ind, col_name in enumerate(spec_ind_labels):
spec_ind_dict[col_name] = (slice(col_ind, col_ind + 1), slice(None))
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', np.uint32(spec_ind_mat))
ds_spec_inds = VirtualDataset('Spectroscopic_Indices', INDICES_DTYPE(spec_ind_mat))
ds_spec_inds.attrs['labels'] = spec_ind_dict
ds_spec_vals = VirtualDataset('Spectroscopic_Values', spec_ind_mat)
ds_spec_vals.attrs['labels'] = spec_ind_dict
......@@ -205,7 +205,7 @@ class SporcTranslator(Translator):
second_path = path.join(fold, 'SPORC_wave.mat')
h5_sporc_parms = h5py.File(second_path, 'r') # Use this for v7.3 and beyond.
excit_wfm = np.squeeze(h5_sporc_parms['FORC_vec'].value)
spec_ind_mat = np.float32(h5_sporc_parms['ind_vecs'].value)
spec_ind_mat = VALUES_DTYPE(h5_sporc_parms['ind_vecs'].value)
h5_sporc_parms.close()
return parm_dict, excit_wfm, spec_ind_mat
......@@ -13,6 +13,7 @@ from scipy.spatial.distance import pdist
from .proc_utils import get_component_slice
from ..core.processing.process import Process
from ..core.io.hdf_utils import get_h5_obj_refs, check_and_link_ancillary, copy_main_attributes
from ..core.io.write_utils import INDICES_DTYPE, VALUES_DTYPE
from ..core.io.hdf_writer import HDFwriter
from ..core.io.dtype_utils import check_dtype, transform_to_target_dtype
from ..core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -185,12 +186,12 @@ class Cluster(Process):
clust_ind_mat = np.transpose(np.atleast_2d(np.arange(num_clusters)))
ds_cluster_inds = VirtualDataset('Cluster_Indices', np.uint32(clust_ind_mat))
ds_cluster_vals = VirtualDataset('Cluster_Values', np.float32(clust_ind_mat))
ds_cluster_inds = VirtualDataset('Cluster_Indices', INDICES_DTYPE(clust_ind_mat))
ds_cluster_vals = VirtualDataset('Cluster_Values', VALUES_DTYPE(clust_ind_mat))
ds_cluster_centroids = VirtualDataset('Mean_Response', mean_response, dtype=mean_response.dtype)
# Main attributes will be copied from h5_main after writing
ds_label_inds = VirtualDataset('Label_Spectroscopic_Indices', np.atleast_2d([0]), dtype=np.uint32)
ds_label_vals = VirtualDataset('Label_Spectroscopic_Values', np.atleast_2d([0]), dtype=np.float32)
ds_label_inds = VirtualDataset('Label_Spectroscopic_Indices', np.atleast_2d([0]), dtype=INDICES_DTYPE)
ds_label_vals = VirtualDataset('Label_Spectroscopic_Values', np.atleast_2d([0]), dtype=VALUES_DTYPE)
# write the labels and the mean response to h5
clust_slices = {'Cluster': (slice(None), slice(0, 1))}
......@@ -213,7 +214,7 @@ class Cluster(Process):
Setup the Spectroscopic Indices and Values for the Mean Response if we didn't use all components
'''
if self.num_comps != self.h5_main.shape[1]:
ds_centroid_indices = VirtualDataset('Mean_Response_Indices', np.arange(self.num_comps, dtype=np.uint32))
ds_centroid_indices = VirtualDataset('Mean_Response_Indices', np.arange(self.num_comps, dtype=INDICES_DTYPE))
if isinstance(self.data_slice[1], np.ndarray):
centroid_vals_mat = h5_spec_vals[self.data_slice[1].tolist()]
......
......@@ -13,6 +13,7 @@ import sklearn.decomposition as dec
from ..core.processing.process import Process
from ..core.io.hdf_utils import get_h5_obj_refs, check_and_link_ancillary
from ..core.io.hdf_writer import HDFwriter
from ..core.io.write_utils import INDICES_DTYPE, VALUES_DTYPE
from ..core.io.dtype_utils import check_dtype, transform_to_target_dtype
from ..core.io.virtual_data import VirtualGroup, VirtualDataset
......@@ -134,8 +135,8 @@ class Decomposition(Process):
decomp_ind_mat = np.transpose(np.atleast_2d(np.arange(components.shape[0])))
ds_decomp_inds = VirtualDataset('Decomposition_Indices', np.uint32(decomp_ind_mat))
ds_decomp_vals = VirtualDataset('Decomposition_Values', np.float32(decomp_ind_mat))
ds_decomp_inds = VirtualDataset('Decomposition_Indices', INDICES_DTYPE(decomp_ind_mat))
ds_decomp_vals = VirtualDataset('Decomposition_Values', VALUES_DTYPE(decomp_ind_mat))
# write the labels and the mean response to h5
decomp_slices = {'Decomp': (slice(None), slice(0, 1))}
......
......@@ -19,7 +19,7 @@ from ..core.io.hdf_utils import get_h5_obj_refs, copy_attributes, link_h5_object
from ..core.io.hdf_writer import HDFwriter
from ..core.io.io_utils import get_available_memory
from ..core.io.virtual_data import VirtualGroup, VirtualDataset
from ..core.io.write_utils import make_indices_matrix, get_aux_dset_slicing
from ..core.io.write_utils import make_indices_matrix, get_aux_dset_slicing, INDICES_DTYPE, VALUES_DTYPE
from .svd_utils import get_component_slice
windata32 = np.dtype({'names': ['Image Data'],
......@@ -413,10 +413,10 @@ class ImageWindow(object):
'''
Set up the HDF5 Group and Datasets for the windowed data
'''
ds_pos_inds = VirtualDataset('Position_Indices', data=win_pos_mat, dtype=np.int32)
ds_pix_inds = VirtualDataset('Spectroscopic_Indices', data=win_pix_mat, dtype=np.int32)
ds_pos_vals = VirtualDataset('Position_Values', data=win_pos_mat, dtype=np.float32)
ds_pix_vals = VirtualDataset('Spectroscopic_Values', data=win_pix_mat, dtype=np.float32)
ds_pos_inds = VirtualDataset('Position_Indices', data=win_pos_mat, dtype=INDICES_DTYPE)
ds_pix_inds = VirtualDataset('Spectroscopic_Indices', data=win_pix_mat, dtype=INDICES_DTYPE)
ds_pos_vals = VirtualDataset('Position_Values', data=win_pos_mat, dtype=VALUES_DTYPE)
ds_pix_vals = VirtualDataset('Spectroscopic_Values', data=win_pix_mat, dtype=VALUES_DTYPE)
pos_labels = get_aux_dset_slicing(['Window Origin X', 'Window Origin Y'], is_spectroscopic=False)
ds_pos_inds.attrs['labels'] = pos_labels
ds_pos_inds.attrs['units'] = ['pixel', 'pixel']
......
......@@ -14,7 +14,7 @@ from ..core.processing.process import Process, parallel_compute
from ..core.io.virtual_data import VirtualDataset, VirtualGroup
from ..core.io.hdf_utils import get_h5_obj_refs, get_auxillary_datasets, copy_attributes, link_as_main, \