Commit 4f34acc2 authored by Somnath, Suhas's avatar Somnath, Suhas
Browse files

Added more translators

Cleaned up more print statements and CamelCased the class names, etc.
parent 0f823266
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 15 11:10:37 2015
@author: Suhas Somnath
"""
import numpy as np
from .hdfutils import getAuxData
def maxReadPixels(max_memory, tot_pix, bins_per_step, bytes_per_bin=4):
"""
Calculates the maximum number of pixels that can be loaded into the
specified memory size. This is particularly useful when applying a
(typically parallel) operation / processing on each pixel.
Example - Fitting response to a model.
Parameters
-------------
max_memory : unsigned int
Maximum memory (in bytes) that can be used.
For example 4 GB would be = 4*((2**10)**3) bytes
tot_pix : unsigned int
Total number of pixels in dataset
bins_per_step : unsigned int
Number of bins that will be read (can be portion of each pixel)
bytes_per_bin : (Optional) unsigned int
size of each bin - set to 4 bytes
Returns
-------
max_pix : unsigned int
Maximum number of pixels that will be loaded
"""
# alternatively try .nbytes
bytes_per_step = bins_per_step * bytes_per_bin
max_pix = np.rint(max_memory / bytes_per_step)
#print('Allowed to read {} of {} pixels'.format(max_pix,tot_pix))
max_pix = max(1,min(tot_pix, max_pix))
return np.uint(max_pix)
def getActiveUDVSsteps(h5_raw):
"""
Returns all the active UDVS steps in the data
Parameters
----------
h5_raw : HDF5 dataset reference
Reference to the raw data
Returns
-----------
steps : 1D numpy array
Active UDVS steps
"""
udvs_step_vec = getAuxData(h5_raw, auxDataName =['UDVS_Indices'])[0].value
return np.unique(udvs_step_vec)
def getSliceForExcWfm(h5_bin_wfm, excit_wfm):
"""
Returns the indices that correspond to the given excitation waveform
that can be used to slice the bin datasets
* Developer note - Replace the first parameter with the Raw_Data dataset
Parameters
----------------
h5_bin_wfm : Reference to HDF5 dataset
Bin Waveform Indices
excit_wfm : integer
excitation waveform / wave type
Returns
--------------
slc : slice object
Slice with the start and end indices
"""
temp = np.where(h5_bin_wfm.value == excit_wfm)[0]
return slice(temp[0],temp[-1]+1) # Need to add one additional index otherwise, the last index will be lost
def getDataIndicesForUDVSstep(h5_udvs_inds, udvs_step_index):
"""
Returns the spectroscopic indices that correspond to the given udvs_step_index
that can be used to slice the main data matrix.
* Developer note - Replace the first parameter with the Raw_Data dataset
Parameters
-------------
h5_udvs_inds : Reference to HDF5 dataset
UDVS_Indices dataset
udvs_step_index : usigned int
UDVS step index (base 0)
Returns
--------------
ans : 1D numpy array
Spectroscopic indices
"""
spec_ind_udvs_step_col = h5_udvs_inds[h5_udvs_inds.attrs.get('UDVS_Step')]
return np.where(spec_ind_udvs_step_col == udvs_step_index)[0]
def getSpecSliceForUDVSstep(h5_udvs_inds, udvs_step_index):
"""
Returns the spectroscopic indices that correspond to the given udvs_step_index
that can be used to slice the main data matrix
* Developer note - Replace the first parameter with the Raw_Data dataset
Parameters
-------------
h5_udvs_inds : Reference to HDF5 dataset
UDVS_Indices dataset
udvs_step_index : unsigned int
UDVS step index (base 0)
Returns
----------
slc : slice object
Object containing the start and end indices
"""
temp = np.where(h5_udvs_inds.value == udvs_step_index)[0]
return slice(temp[0],temp[-1]+1) # Need to add one additional index otherwise, the last index will be lost
def getForExcitWfm(h5_main, h5_other, wave_type):
"""
Slices the provided H5 dataset by the provided wave type.
Note that this is applicable to only certain H5 datasets such as the bin frequences, bin FFT etc.
Parameters
----------
h5_main : Reference to HDF5 dataset
Raw_Data dataset
h5_other :Reference to HDF5 dataset
The dataset that needs to be sliced such as bin frequencies
wave_type : unsigned int
Excitation waveform type
Returns
---------
freq_vec : 1D numpy array
data specific to specified excitation waveform
"""
h5_bin_wfm_type = getAuxData(h5_main, auxDataName=['Bin_Wfm_Type'])[0]
inds = np.where(h5_bin_wfm_type.value == wave_type)[0]
return h5_other[slice(inds[0],inds[-1]+1)]
def getIndicesforPlotGroup(h5_udvs_inds, ds_udvs, plt_grp_name):
"""
For a provided plot group name in the udvs table, this function
returns the corresponding spectroscopic indices that can be used to index / slice the main data set
and the data within the udvs table for the requested plot group
* Developer note - Replace the first parameter with the Raw_Data dataset
Parameters
------------
h5_udvs_inds : Reference to HDF5 dataset
containing the UDVS indices
ds_udvs : Reference to HDF5 dataset
containing the UDVS table
plt_grp_name : string
name of the plot group in the UDVS table
Returns
-----------
step_bin_indices : 2D numpy array
Indices arranged as [step, bin] in the spectroscopic_indices table
This is useful for knowing the number of bins and steps in this plot group.
We are allowed to assume that the number of bins does NOT change within the plot group
oneD_indices : 1D numpy array
spectroscopic indices corresponding to the requested plot group
udvs_plt_grp_col : 1D numpy array
data contained within the udvs table for the requested plot group
"""
# working on the UDVS table first:
# getting the numpy array corresponding the requested plot group
udvs_col_data = np.squeeze(ds_udvs[ds_udvs.attrs.get(plt_grp_name)])
# All UDVS steps that are NOT part of the plot grop are empty cells in the table
# and hence assume a nan value.
# getting the udvs step indices that belong to this plot group:
step_inds = np.where(np.isnan(udvs_col_data) == False)[0]
# Getting the values in that plot group that were non NAN
udvs_plt_grp_col = udvs_col_data[step_inds]
#---------------------------------
# Now we use the udvs step indices calculated above to get
# the indices in the spectroscopic indices table
spec_ind_udvs_step_col = h5_udvs_inds[h5_udvs_inds.attrs.get('UDVS_Step')]
num_bins = len(np.where(spec_ind_udvs_step_col == step_inds[0])[0])
# Stepehen says that we can assume that the number of bins will NOT change in a plot group
step_bin_indices = np.zeros(shape=(len(step_inds),num_bins), dtype=int)
for indx, step in enumerate(step_inds):
step_bin_indices[indx,:] = np.where(spec_ind_udvs_step_col == step)[0]
oneD_indices = step_bin_indices.reshape((step_bin_indices.shape[0]*step_bin_indices.shape[1]))
return (step_bin_indices, oneD_indices, udvs_plt_grp_col)
def reshapeToOneStep(raw_mat, num_steps):
"""
Reshapes provided data from (pos, step * bin) to (pos * step, bin).
This is useful when unraveling data for parallel processing.
Parameters
-------------
raw_mat : 2D numpy array
Data organized as (positions, step * bins)
num_steps : unsigned int
Number of spectroscopic steps per pixel (eg - UDVS steps)
Returns
--------------
twoD : 2D numpy array
Data rearranged as (positions * step, bin)
"""
num_pos = raw_mat.shape[0]
num_bins = int(raw_mat.shape[1]/num_steps)
oneD = raw_mat
oneD = oneD.reshape((num_bins * num_steps * num_pos))
twoD = oneD.reshape((num_steps * num_pos, num_bins))
return twoD
def reshapeToNsteps(raw_mat, num_steps):
"""
Reshapes provided data from (positions * step, bin) to (positions, step * bin).
Use this to restructure data back to its original form after parallel computing
Parameters
--------------
raw_mat : 2D numpy array
Data organized as (positions * step, bin)
num_steps : unsigned int
Number of spectroscopic steps per pixel (eg - UDVS steps)
Returns
---------------
twoD : 2D numpy array
Data rearranged as (positions, step * bin)
"""
num_bins = raw_mat.shape[1]
num_pos = int(raw_mat.shape[0]/num_steps)
oneD = raw_mat
oneD = oneD.reshape(num_bins * num_steps * num_pos)
twoD = oneD.reshape((num_pos, num_steps * num_bins))
return twoD
def generateTestSpectroscopicData(num_bins=7, num_steps=3, num_pos=4):
"""
Generates a (preferably small) test data set using the given parameters.
Data is filled with indices (with base 1 for simplicity).
Use this for testing reshape operations etc.
Parameters
----------
num_bins : unsigned int (Optional. Default = 7)
Number of bins
num_steps : unsigned int (Optional. Default = 3)
Number of spectroscopic steps
num_pos : unsigned int (Optional. Default = 4)
Number of fictional positions
Returns
--------------
full_data : 2D numpy array
Data organized as [steps x bins, positions]
"""
full_data = np.zeros((num_steps * num_bins, num_pos));
for pos in xrange(num_pos):
bin_count=0
for step in xrange(num_steps):
for bind in xrange(num_bins):
full_data[bin_count,pos] = (pos+1)*100 + (step+1)*10 + (bind+1)
bin_count+=1
return full_data
def isSimpleDataset(h5_main, isBEPS=True):
"""
This function figures out if a single number defines the bins for all UDVS steps
In such cases (udvs_steps x bins, pos) can be reshaped to (bins, positions x steps)
for (theoretically) faster computation, especially for large datasets
Actually, things are a lot simpler. Only need to check if number of bins for all excitation waveforms are equal
Parameters
-------------
h5_main : Reference to HDF5 dataset
Raw_Data dataset
isBEPS : Boolean (default = True)
Whether or not this dataset is BEPS
Returns
----------
data_type : Boolean
Whether or not this dataset can be unraveled / flattened
"""
if isBEPS:
if h5_main.parent.parent.attrs['VS_mode'] in ['DC modulation mode','AC modulation mode with time reversal','current mode','Relaxation']:
# I am pretty sure that AC modulation also is simple
return True
else:
# Could be user defined or some other kind I am not aware of
# In many cases, some of these datasets could also potentially be simple datasets
ds_udvs = getAuxData(h5_main, auxDataName=['UDVS'])[0]
excit_wfms = ds_udvs[ds_udvs.attrs.get('wave_mod')]
wfm_types = np.unique(excit_wfms)
if len(wfm_types) == 1:
# BEPS with single excitation waveform
print('Single BEPS excitation waveform')
return True
else:
# Multiple waveform types here
harm_types = np.unique(np.abs(wfm_types))
if len(harm_types) != 1:
# eg - excitaiton waveforms 1, 2, 3 NOT -1, +1
return False
# In this case a single excitation waveform with forward and reverse was used.
h5_bin_wfm_type = getAuxData(h5_main, auxDataName=['Bin_Wfm_Type'])[0]
# Now for each wfm type, count number of bins.
wfm_bin_count = []
for wfm in wfm_types:
wfm_bin_count.append(np.where(h5_bin_wfm_type.value == wfm)[0])
wfm_lengths = np.unique(np.array(wfm_bin_count))
if len(wfm_lengths) == 1:
# BEPS with multiple excitation waveforms but each excitation waveform has same number of bins
print('All BEPS excitation waves have same number of bins')
return True
return False
else:
# BE-Line
return True
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -10,13 +10,13 @@ import numpy as np # For array operations
from os import path, remove # File Path formatting
from warnings import warn
import h5py
from .Translator import Translator
from .Utils import makePositionMat, getPositionSlicing, generateDummyMainParms
from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..ioUtils import getH5DsetRefs
from .translator import Translator
from .utils import makePositionMat, getPositionSlicing, generateDummyMainParms
from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..hdfutils import getH5DsetRefs
class FastIV(Translator):
class FastIVTranslator(Translator):
"""
Translates G-mode Fast IV datasets from .mat files to .h5
"""
......
......@@ -5,18 +5,18 @@ Created on Fri Mar 04 11:12:45 2016
@author: Suhas Somnath
"""
from __future__ import print_function, division; # int/int = float
import numpy as np; # For array operations
from __future__ import print_function, division # int/int = float
import numpy as np # For array operations
from os import path, remove# File Path formatting
from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
from .Translator import Translator # Because this class extends the abstract Translator class
from .Utils import makePositionMat, getPositionSlicing, generateDummyMainParms
from .GmodeUtils import readGmodeParms
from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..ioHDF5 import getH5DsetRefs
from .translator import Translator # Because this class extends the abstract Translator class
from .utils import makePositionMat, getPositionSlicing, generateDummyMainParms
from .gmode_utils import readGmodeParms
from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..hdfutils import getH5DsetRefs
class GeneralDynamicMode(Translator):
class GDMTranslator(Translator):
"""
Translates G-mode w^2 datasets from .mat files to .h5
"""
......
......@@ -9,14 +9,14 @@ import numpy as np # For array operations
from os import path, listdir, remove # File Path formatting
from warnings import warn
from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
from .Translator import Translator
from .Utils import interpretFreq, makePositionMat, getPositionSlicing, generateDummyMainParms
from .BEutils import parmsToDict
from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..ioUtils import getH5DsetRefs
from .translator import Translator
from .utils import interpretFreq, makePositionMat, getPositionSlicing, generateDummyMainParms
from .beutils import parmsToDict
from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
from ..hdfutils import getH5DsetRefs
class GLine(Translator):
class GLineTranslator(Translator):
"""
Translated G-mode line (bigtimedata.dat) files from actual BE line experiments to HDF5
"""
......
# -*- coding: utf-8 -*-
"""
Created on Fri Mar 04 11:16:20 2016
@author: Suhas Somnath
"""
from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
import numpy as np
def readGmodeParms(parm_path):
"""
Translates the parameters stored in a G-mode parms.mat file into a python dictionary
Parameters
------------
parm_path : String / unicode
Absolute path of the parms.mat file
Returns
----------
parm_dict : Dictionary
dictionary of relevant parameters neatly formatted with units
"""
parm_data = loadmat(parm_path, squeeze_me=True, struct_as_record=True)
parm_dict = dict()
IO_parms = parm_data['IOparms']
parm_dict['IO_samp_rate_[Hz]'] = np.int32(IO_parms['sampRate'].item())
parm_dict['IO_down_samp_rate_[Hz]'] = np.int32(IO_parms['downSampRate'].item())
parm_dict['IO_AO0_amp'] = np.int32(IO_parms['AO0_amp'].item())
parm_dict['IO_AO1_amp'] = np.int32(IO_parms['AO1_amp'].item())
parm_dict['IO_AI_chans'] = np.int32(parm_data['aiChans'])
env_parms = parm_data['envParms']
parm_dict['envelope_mode'] = np.int32(env_parms['envMode'].item())
parm_dict['envelope_type'] = np.int32(env_parms['envType'].item())
parm_dict['envelope_smoothing'] = np.int32(env_parms['smoothing'].item())
forc_parms = parm_data['forcParms']
parm_dict['FORC_V_high_1_[V]'] = np.float(forc_parms['vHigh1'].item())
parm_dict['FORC_V_high_2_[V]'] = np.float(forc_parms['vHigh2'].item())
parm_dict['FORC_V_low_1_[V]'] = np.float(forc_parms['vLow1'].item())
parm_dict['FORC_V_low_2_[V]'] = np.float(forc_parms['vLow2'].item())
gen_sig = parm_data['genSig']
parm_dict['wfm_f_fast_[Hz]'] = np.float(gen_sig['fFast'].item())
parm_dict['wfm_d_fast_[s]'] = np.float(gen_sig['dFast'].item())
parm_dict['wfm_p_slow_[s]'] = np.float(gen_sig['pSlow'].item())
parm_dict['wfm_n_cycles'] = np.int32(gen_sig['nCycles'].item())
parm_dict['wfm_swap_mode'] = np.int32(gen_sig['swapMode'].item())
parm_dict['wfm_reps'] = np.int32(gen_sig['mReps'].item())
scl_parms = parm_data['sclParms']
parm_dict['wfm_amp_tip_fast_[V]'] = np.float(scl_parms['ampTipFast'].item())
parm_dict['wfm_off_tip_fast_[V]'] = np.float(scl_parms['offTipFast'].item())
parm_dict['wfm_amp_tip_slow_[V]'] = np.float(scl_parms['ampTipSlow'].item())
parm_dict['wfm_off_tip_slow_[V]'] = np.float(scl_parms['offTipSlow'].item())
parm_dict['wfm_amp_BD_fast_[V]'] = np.float(scl_parms['ampBDfast'].item())
parm_dict['wfm_off_BD_fast_[V]'] = np.float(scl_parms['offBDfast'].item())
parm_dict['grid_num_rows'] = parm_data['numrows']
parm_dict['grid_num_cols'] = parm_data['numcols']
return parm_dict
\ No newline at end of file
"""
Created on Feb 9, 2016
@author: Chris Smith
"""
import numpy as np
import os
from scipy.misc import imread
from ..iohdf5 import ioHDF5
from ..microdata import MicroDataGroup, MicroDataset
from ..hdfutils import getH5DsetRefs
from .translator import Translator
from .utils import generateDummyMainParms, makePositionMat, getPositionSlicing
class PtychographyTranslator(Translator):
"""
Translate Pytchography data from a set of images to an HDF5 file
"""
def translate(self, h5_path, image_path):
"""
Basic method that adds Ptychography data to existing hdf5 thisfile
You must have already done the basic translation with BEodfTranslator
Parameters
----------------
h5_path : Absolute thisfile path for the data thisfile.
Must be in hdf5 format
image_path : Absolute path to folder holding the image files
Returns
----------
None
"""
# Get the list of all files with the .tif extension and the number of files in the list
file_list, num_files = self.__getfilelist(image_path,'.tif')
# Open the hdf5 thisfile and delete any contents
try:
hdf = ioHDF5(h5_path)
hdf.clear()
except:
raise
# Set up the basic parameters associated with this set of images
(usize,vsize),data_type = self.__getimagesize(os.path.join(image_path,file_list[0]))
num_pixels = usize*vsize
scan_size = np.int(np.sqrt(num_files))
num_files = scan_size**2
mean_ronch = np.zeros(num_pixels,dtype=np.float32)
h5_main, h5_mean_spec, h5_ronch = self.__setupH5(num_files, hdf, usize,
vsize, np.float32,
num_pixels,
scan_size)
for ifile, thisfile in enumerate(file_list[:num_files]):
selected = (ifile+1) % round(num_files/16) == 0
if selected:
print('Processing file...{}% - reading: {}'.format(round(100*ifile/num_files),thisfile))
image = imread(os.path.join(image_path,thisfile))
image = image.reshape(num_pixels)
h5_main[ifile,:] = image
h5_mean_spec[ifile] = np.mean(image)
mean_ronch += image
hdf.flush()
#print('{}, {}'.format(np.max(mean_ronch), np.min(mean_ronch)))
h5_ronch[:] = mean_ronch/num_files
hdf.flush()
hdf.close()
def __getfilelist(self, path, ftype='all'):
"""
Returns a list of all files in the directory given by path
Parameters
---------------
path : string / unicode
absolute path to directory containing files
ftype : this file types to return in file_list. (optional. Default is all)
Returns
----------
file_list : list of strings
names of all files in directory located at path
numfiles : unsigned int
number of files in file_list
"""
# Get all files in directory
file_list = os.listdir(path)
# If no file type specified, return full list
if ftype=='all':
numfiles = len(file_list)
return (file_list, numfiles)
# Remove files of type other than the request ftype from the list
new_file_list = []
for this_thing in file_list:
split = os.path.splitext(this_thing)
if len(split) <2:
continue
ext = split[1]
if ext == ftype:
new_file_list.append(this_thing)
numfiles = len(new_file_list)
return (new_file_list,numfiles)