Added more translators

Cleaned up more print statements and CamelCased the class names, etc.

pycroscopy/io/be_hdf_utils.py

+# -*- coding: utf-8 -*-
+"""
+Created on Tue Dec 15 11:10:37 2015
+
+@author: Suhas Somnath
+"""
+import numpy as np
+from .hdfutils import getAuxData
+
+def maxReadPixels(max_memory, tot_pix, bins_per_step, bytes_per_bin=4):
+    """
+    Calculates the maximum number of pixels that can be loaded into the 
+    specified memory size. This is particularly useful when applying a 
+    (typically parallel) operation / processing on each pixel. 
+    Example - Fitting response to a model.
+    
+    Parameters
+    -------------
+    max_memory : unsigned int
+        Maximum memory (in bytes) that can be used. 
+        For example 4 GB would be = 4*((2**10)**3) bytes
+    tot_pix : unsigned int
+        Total number of pixels in dataset
+    bins_per_step : unsigned int
+        Number of bins that will be read (can be portion of each pixel)
+    bytes_per_bin : (Optional) unsigned int
+        size of each bin - set to 4 bytes
+    
+    Returns
+    -------
+    max_pix : unsigned int
+        Maximum number of pixels that will be loaded
+    """
+    # alternatively try .nbytes
+    bytes_per_step = bins_per_step * bytes_per_bin
+    max_pix = np.rint(max_memory / bytes_per_step)
+    #print('Allowed to read {} of {} pixels'.format(max_pix,tot_pix))
+    max_pix = max(1,min(tot_pix, max_pix))
+    return np.uint(max_pix)
+
+def getActiveUDVSsteps(h5_raw):
+    """
+    Returns all the active UDVS steps in the data
+    
+    Parameters
+    ----------
+    h5_raw : HDF5 dataset reference
+        Reference to the raw data
+        
+    Returns
+    -----------
+    steps : 1D numpy array
+        Active UDVS steps
+    """
+    udvs_step_vec = getAuxData(h5_raw, auxDataName =['UDVS_Indices'])[0].value
+    return np.unique(udvs_step_vec)
+    
+def getSliceForExcWfm(h5_bin_wfm, excit_wfm):
+    """
+    Returns the indices that correspond to the given excitation waveform
+    that can be used to slice the bin datasets
+    * Developer note - Replace the first parameter with the Raw_Data dataset
+    
+    Parameters
+    ----------------
+    h5_bin_wfm : Reference to HDF5 dataset
+        Bin Waveform Indices
+    excit_wfm : integer
+        excitation waveform / wave type
+    
+    Returns
+    --------------
+    slc : slice object
+        Slice with the start and end indices
+    """
+    temp = np.where(h5_bin_wfm.value == excit_wfm)[0]
+    return slice(temp[0],temp[-1]+1) # Need to add one additional index otherwise, the last index will be lost
+    
+def getDataIndicesForUDVSstep(h5_udvs_inds, udvs_step_index):
+    """
+    Returns the spectroscopic indices that correspond to the given udvs_step_index
+    that can be used to slice the main data matrix.
+    * Developer note - Replace the first parameter with the Raw_Data dataset
+    
+    Parameters
+    -------------
+    h5_udvs_inds : Reference to HDF5 dataset 
+        UDVS_Indices dataset
+    udvs_step_index : usigned int
+        UDVS step index (base 0)
+    
+    Returns
+    --------------
+    ans : 1D numpy array 
+        Spectroscopic indices
+    """
+    spec_ind_udvs_step_col = h5_udvs_inds[h5_udvs_inds.attrs.get('UDVS_Step')]
+    return np.where(spec_ind_udvs_step_col == udvs_step_index)[0]
+    
+def getSpecSliceForUDVSstep(h5_udvs_inds, udvs_step_index):
+    """
+    Returns the spectroscopic indices that correspond to the given udvs_step_index
+    that can be used to slice the main data matrix
+    * Developer note - Replace the first parameter with the Raw_Data dataset
+    
+    Parameters
+    -------------
+    h5_udvs_inds : Reference to HDF5 dataset
+        UDVS_Indices dataset
+    udvs_step_index : unsigned int
+        UDVS step index (base 0)
+    
+    Returns
+    ----------
+    slc : slice object
+        Object containing the start and end indices
+    """
+    temp = np.where(h5_udvs_inds.value == udvs_step_index)[0]
+    return slice(temp[0],temp[-1]+1) # Need to add one additional index otherwise, the last index will be lost
+    
+
+def getForExcitWfm(h5_main, h5_other, wave_type):
+    """
+    Slices the provided H5 dataset by the provided wave type. 
+    Note that this is applicable to only certain H5 datasets such as the bin frequences, bin FFT etc.
+    
+    Parameters
+    ----------
+    h5_main : Reference to HDF5 dataset 
+        Raw_Data dataset
+    h5_other :Reference to HDF5 dataset 
+        The dataset that needs to be sliced such as bin frequencies
+    wave_type : unsigned int
+        Excitation waveform type
+
+    Returns
+    ---------
+    freq_vec : 1D numpy array
+        data specific to specified excitation waveform
+    """    
+    h5_bin_wfm_type = getAuxData(h5_main, auxDataName=['Bin_Wfm_Type'])[0]
+    inds = np.where(h5_bin_wfm_type.value == wave_type)[0]
+    return h5_other[slice(inds[0],inds[-1]+1)]
+
+    
+def getIndicesforPlotGroup(h5_udvs_inds, ds_udvs, plt_grp_name):
+    """
+    For a provided plot group name in the udvs table, this function 
+    returns the corresponding spectroscopic indices that can be used to index / slice the main data set
+    and the data within the udvs table for the requested plot group
+    * Developer note - Replace the first parameter with the Raw_Data dataset
+        
+    Parameters
+    ------------
+    h5_udvs_inds : Reference to HDF5 dataset
+        containing the UDVS indices
+    ds_udvs : Reference to HDF5 dataset 
+        containing the UDVS table
+    plt_grp_name : string 
+        name of the plot group in the UDVS table
+            
+    Returns
+    -----------
+    step_bin_indices : 2D numpy array
+        Indices arranged as [step, bin] in the spectroscopic_indices table
+        This is useful for knowing the number of bins and steps in this plot group.
+        We are allowed to assume that the number of bins does NOT change within the plot group
+    oneD_indices : 1D numpy array
+        spectroscopic indices corresponding to the requested plot group
+    udvs_plt_grp_col : 1D numpy array
+        data contained within the udvs table for the requested plot group        
+    """
+    
+    # working on the UDVS table first:
+    # getting the numpy array corresponding the requested plot group
+    udvs_col_data = np.squeeze(ds_udvs[ds_udvs.attrs.get(plt_grp_name)])
+    # All UDVS steps that are NOT part of the plot grop are empty cells in the table
+    # and hence assume a nan value.
+    # getting the udvs step indices that belong to this plot group:
+    step_inds = np.where(np.isnan(udvs_col_data) == False)[0]    
+    # Getting the values in that plot group that were non NAN
+    udvs_plt_grp_col = udvs_col_data[step_inds]
+    
+    #---------------------------------
+    
+    # Now we use the udvs step indices calculated above to get 
+    # the indices in the spectroscopic indices table
+    spec_ind_udvs_step_col = h5_udvs_inds[h5_udvs_inds.attrs.get('UDVS_Step')]
+    num_bins = len(np.where(spec_ind_udvs_step_col == step_inds[0])[0])
+    # Stepehen says that we can assume that the number of bins will NOT change in a plot group
+    step_bin_indices = np.zeros(shape=(len(step_inds),num_bins), dtype=int)
+
+    for indx, step in enumerate(step_inds):
+        step_bin_indices[indx,:] = np.where(spec_ind_udvs_step_col == step)[0]
+    
+    oneD_indices = step_bin_indices.reshape((step_bin_indices.shape[0]*step_bin_indices.shape[1]))
+    return (step_bin_indices, oneD_indices, udvs_plt_grp_col)
+    
+def reshapeToOneStep(raw_mat, num_steps):
+    """
+    Reshapes provided data from (pos, step * bin) to (pos * step, bin). 
+    This is useful when unraveling data for parallel processing.
+    
+    Parameters
+    -------------
+    raw_mat : 2D numpy array
+        Data organized as (positions, step * bins)
+    num_steps : unsigned int
+        Number of spectroscopic steps per pixel (eg - UDVS steps)
+        
+    Returns
+    --------------
+    twoD : 2D numpy array
+        Data rearranged as (positions * step, bin)
+    """
+    num_pos = raw_mat.shape[0]
+    num_bins = int(raw_mat.shape[1]/num_steps)
+    oneD = raw_mat
+    oneD = oneD.reshape((num_bins * num_steps * num_pos))
+    twoD = oneD.reshape((num_steps * num_pos, num_bins))
+    return twoD
+    
+def reshapeToNsteps(raw_mat, num_steps):
+    """
+    Reshapes provided data from (positions * step, bin) to (positions, step * bin).  
+    Use this to restructure data back to its original form after parallel computing 
+    
+    Parameters
+    --------------
+    raw_mat : 2D numpy array 
+        Data organized as (positions * step, bin)
+    num_steps : unsigned int
+         Number of spectroscopic steps per pixel (eg - UDVS steps)
+        
+    Returns
+    --------------- 
+    twoD : 2D numpy array
+        Data rearranged as (positions, step * bin)
+    """
+    num_bins = raw_mat.shape[1]
+    num_pos = int(raw_mat.shape[0]/num_steps)
+    oneD = raw_mat
+    oneD = oneD.reshape(num_bins * num_steps * num_pos)
+    twoD = oneD.reshape((num_pos, num_steps * num_bins)) 
+    return twoD
+    
+def generateTestSpectroscopicData(num_bins=7, num_steps=3, num_pos=4):
+    """
+    Generates a (preferably small) test data set using the given parameters.
+    Data is filled with indices (with base 1 for simplicity). 
+    Use this for testing reshape operations etc.
+    
+    Parameters
+    ----------
+    num_bins : unsigned int (Optional. Default = 7)
+        Number of bins
+    num_steps : unsigned int (Optional. Default = 3)
+        Number of spectroscopic steps  
+    num_pos : unsigned int (Optional. Default = 4)
+        Number of fictional positions
+    
+    Returns
+    --------------
+    full_data : 2D numpy array
+        Data organized as [steps x bins, positions]
+    """
+    full_data = np.zeros((num_steps * num_bins, num_pos));
+    for pos in xrange(num_pos):
+        bin_count=0
+        for step in xrange(num_steps):
+            for bind in xrange(num_bins):
+                full_data[bin_count,pos] = (pos+1)*100 + (step+1)*10 + (bind+1)
+                bin_count+=1
+    return full_data
+
+
+def isSimpleDataset(h5_main, isBEPS=True):
+    """
+    This function figures out if a single number defines the bins for all UDVS steps
+    In such cases (udvs_steps x bins, pos) can be reshaped to (bins, positions x steps)
+    for (theoretically) faster computation, especially for large datasets
+
+    Actually, things are a lot simpler. Only need to check if number of bins for all excitation waveforms are equal
+    
+    Parameters
+    -------------
+    h5_main : Reference to HDF5 dataset
+        Raw_Data dataset
+    isBEPS : Boolean (default = True)
+        Whether or not this dataset is BEPS
+        
+    Returns
+    ----------
+    data_type : Boolean
+        Whether or not this dataset can be unraveled / flattened
+    """
+    
+    if isBEPS:
+        if h5_main.parent.parent.attrs['VS_mode'] in ['DC modulation mode','AC modulation mode with time reversal','current mode','Relaxation']:
+            # I am pretty sure that AC modulation also is simple
+            return True
+        else:
+            # Could be user defined or some other kind I am not aware of
+            # In many cases, some of these datasets could also potentially be simple datasets
+            ds_udvs = getAuxData(h5_main, auxDataName=['UDVS'])[0]                
+            excit_wfms = ds_udvs[ds_udvs.attrs.get('wave_mod')]
+            wfm_types = np.unique(excit_wfms)
+            if len(wfm_types) == 1:
+                # BEPS with single excitation waveform
+                print('Single BEPS excitation waveform')
+                return True
+            else:
+                # Multiple waveform types here
+                harm_types = np.unique(np.abs(wfm_types))
+                if len(harm_types) != 1:
+                    # eg - excitaiton waveforms 1, 2, 3 NOT -1, +1
+                    return False
+                # In this case a single excitation waveform with forward and reverse was used.
+                h5_bin_wfm_type = getAuxData(h5_main, auxDataName=['Bin_Wfm_Type'])[0]
+                # Now for each wfm type, count number of bins.
+                wfm_bin_count = []
+                for wfm in wfm_types:
+                    wfm_bin_count.append(np.where(h5_bin_wfm_type.value == wfm)[0])
+                wfm_lengths = np.unique(np.array(wfm_bin_count))
+                if len(wfm_lengths) == 1:
+                    # BEPS with multiple excitation waveforms but each excitation waveform has same number of bins
+                    print('All BEPS excitation waves have same number of bins')
+                    return True
+            return False   
+    else:
+        # BE-Line
+        return True    
\ No newline at end of file

pycroscopy/io/translators/fastiv.py

@@ -10,13 +10,13 @@ import numpy as np # For array operations
 from os import path, remove # File Path formatting
 from warnings import warn
 import h5py
-from .Translator import Translator
-from .Utils import makePositionMat, getPositionSlicing, generateDummyMainParms
-from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
-from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
-from ..ioUtils import getH5DsetRefs
+from .translator import Translator
+from .utils import makePositionMat, getPositionSlicing, generateDummyMainParms
+from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
+from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
+from ..hdfutils import getH5DsetRefs
 
-class FastIV(Translator):
+class FastIVTranslator(Translator):
     """
     Translates G-mode Fast IV datasets from .mat files to .h5
     """

pycroscopy/io/translators/generaldynamicmode.py → pycroscopy/io/translators/general_dynamic_mode.py

@@ -5,18 +5,18 @@ Created on Fri Mar 04 11:12:45 2016
 @author: Suhas Somnath
 """
 
-from __future__ import print_function, division; # int/int = float
-import numpy as np; # For array operations
+from __future__ import print_function, division # int/int = float
+import numpy as np # For array operations
 from os import path, remove# File Path formatting
 from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
-from .Translator import Translator # Because this class extends the abstract Translator class
-from .Utils import makePositionMat, getPositionSlicing, generateDummyMainParms 
-from .GmodeUtils import readGmodeParms
-from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
-from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
-from ..ioHDF5 import getH5DsetRefs
+from .translator import Translator # Because this class extends the abstract Translator class
+from .utils import makePositionMat, getPositionSlicing, generateDummyMainParms
+from .gmode_utils import readGmodeParms
+from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
+from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
+from ..hdfutils import getH5DsetRefs
 
-class GeneralDynamicMode(Translator):
+class GDMTranslator(Translator):
     """
     Translates G-mode w^2 datasets from .mat files to .h5
     """

pycroscopy/io/translators/gline.py

@@ -9,14 +9,14 @@ import numpy as np # For array operations
 from os import path, listdir, remove # File Path formatting
 from warnings import warn
 from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
-from .Translator import Translator
-from .Utils import interpretFreq, makePositionMat, getPositionSlicing, generateDummyMainParms
-from .BEutils import parmsToDict
-from ..SPMnode import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
-from ..ioHDF5 import ioHDF5 # Now the translator is responsible for writing the data.
-from ..ioUtils import getH5DsetRefs
+from .translator import Translator
+from .utils import interpretFreq, makePositionMat, getPositionSlicing, generateDummyMainParms
+from .beutils import parmsToDict
+from ..microdata import MicroDataGroup, MicroDataset # The building blocks for defining heirarchical storage in the H5 file
+from ..iohdf5 import ioHDF5 # Now the translator is responsible for writing the data.
+from ..hdfutils import getH5DsetRefs
 
-class GLine(Translator):
+class GLineTranslator(Translator):
     """
     Translated G-mode line (bigtimedata.dat) files from actual BE line experiments to HDF5
     """

pycroscopy/io/translators/gmode_utils.py

+# -*- coding: utf-8 -*-
+"""
+Created on Fri Mar 04 11:16:20 2016
+
+@author: Suhas Somnath
+"""
+
+from scipy.io.matlab import loadmat # To load parameters stored in Matlab .mat file
+import numpy as np
+
+def readGmodeParms(parm_path):
+    """
+    Translates the parameters stored in a G-mode parms.mat file into a python dictionary
+    
+    Parameters
+    ------------
+    parm_path : String / unicode
+        Absolute path of the parms.mat file
+    
+    Returns
+    ----------
+    parm_dict : Dictionary 
+        dictionary of relevant parameters neatly formatted with units
+    """
+    parm_data = loadmat(parm_path, squeeze_me=True, struct_as_record=True)
+    
+    parm_dict = dict()
+    IO_parms = parm_data['IOparms']
+    parm_dict['IO_samp_rate_[Hz]'] = np.int32(IO_parms['sampRate'].item())
+    parm_dict['IO_down_samp_rate_[Hz]'] = np.int32(IO_parms['downSampRate'].item())
+    parm_dict['IO_AO0_amp'] = np.int32(IO_parms['AO0_amp'].item())
+    parm_dict['IO_AO1_amp'] = np.int32(IO_parms['AO1_amp'].item())
+    parm_dict['IO_AI_chans'] = np.int32(parm_data['aiChans'])
+    
+    env_parms = parm_data['envParms']
+    parm_dict['envelope_mode'] = np.int32(env_parms['envMode'].item())
+    parm_dict['envelope_type'] = np.int32(env_parms['envType'].item())
+    parm_dict['envelope_smoothing'] = np.int32(env_parms['smoothing'].item())
+    
+    forc_parms = parm_data['forcParms']
+    parm_dict['FORC_V_high_1_[V]'] = np.float(forc_parms['vHigh1'].item())
+    parm_dict['FORC_V_high_2_[V]'] = np.float(forc_parms['vHigh2'].item())
+    parm_dict['FORC_V_low_1_[V]'] = np.float(forc_parms['vLow1'].item())
+    parm_dict['FORC_V_low_2_[V]'] = np.float(forc_parms['vLow2'].item())
+    
+    gen_sig = parm_data['genSig']
+    parm_dict['wfm_f_fast_[Hz]'] = np.float(gen_sig['fFast'].item())
+    parm_dict['wfm_d_fast_[s]'] = np.float(gen_sig['dFast'].item())
+    parm_dict['wfm_p_slow_[s]'] = np.float(gen_sig['pSlow'].item())
+    parm_dict['wfm_n_cycles'] = np.int32(gen_sig['nCycles'].item())
+    parm_dict['wfm_swap_mode'] = np.int32(gen_sig['swapMode'].item())
+    parm_dict['wfm_reps'] = np.int32(gen_sig['mReps'].item())
+    
+    scl_parms = parm_data['sclParms']
+    parm_dict['wfm_amp_tip_fast_[V]'] = np.float(scl_parms['ampTipFast'].item())
+    parm_dict['wfm_off_tip_fast_[V]'] = np.float(scl_parms['offTipFast'].item())
+    parm_dict['wfm_amp_tip_slow_[V]'] = np.float(scl_parms['ampTipSlow'].item())
+    parm_dict['wfm_off_tip_slow_[V]'] = np.float(scl_parms['offTipSlow'].item())
+    parm_dict['wfm_amp_BD_fast_[V]'] = np.float(scl_parms['ampBDfast'].item())
+    parm_dict['wfm_off_BD_fast_[V]'] = np.float(scl_parms['offBDfast'].item())
+    
+    parm_dict['grid_num_rows'] = parm_data['numrows']
+    parm_dict['grid_num_cols'] = parm_data['numcols']
+    
+    return parm_dict
\ No newline at end of file

pycroscopy/io/translators/ptychography.py

+"""
+Created on Feb 9, 2016
+
+@author: Chris Smith
+"""
+
+import numpy as np
+import os
+from scipy.misc import imread
+from ..iohdf5 import ioHDF5
+from ..microdata import MicroDataGroup, MicroDataset
+from ..hdfutils import getH5DsetRefs
+from .translator import Translator
+from .utils import generateDummyMainParms, makePositionMat, getPositionSlicing
+
+class PtychographyTranslator(Translator):
+    """
+    Translate Pytchography data from a set of images to an HDF5 file
+    """
+
+    def translate(self, h5_path, image_path):
+        """
+        Basic method that adds Ptychography data to existing hdf5 thisfile
+        You must have already done the basic translation with BEodfTranslator
+        
+        Parameters
+        ----------------
+        h5_path : Absolute thisfile path for the data thisfile.
+            Must be in hdf5 format
+        image_path : Absolute path to folder holding the image files 
+            
+        Returns
+        ----------
+        None
+        """
+                
+        # Get the list of all files with the .tif extension and the number of files in the list
+
+        file_list, num_files = self.__getfilelist(image_path,'.tif')
+          
+        # Open the hdf5 thisfile and delete any contents
+        try:
+            hdf = ioHDF5(h5_path)
+            hdf.clear()
+        except:
+            raise
+                
+        # Set up the basic parameters associated with this set of images
+        
+        (usize,vsize),data_type = self.__getimagesize(os.path.join(image_path,file_list[0]))
+        num_pixels = usize*vsize
+        
+        scan_size = np.int(np.sqrt(num_files))
+        num_files = scan_size**2
+        
+        mean_ronch = np.zeros(num_pixels,dtype=np.float32)  
+        
+        h5_main, h5_mean_spec, h5_ronch = self.__setupH5(num_files, hdf, usize, 
+                                                         vsize, np.float32, 
+                                                         num_pixels, 
+                                                         scan_size)
+
+        for ifile, thisfile in enumerate(file_list[:num_files]):
+
+            selected = (ifile+1) % round(num_files/16) == 0
+            if selected:
+                print('Processing file...{}% - reading: {}'.format(round(100*ifile/num_files),thisfile))
+            
+            image = imread(os.path.join(image_path,thisfile))
+            image = image.reshape(num_pixels)
+            h5_main[ifile,:] = image
+            
+            h5_mean_spec[ifile] = np.mean(image)
+            
+            mean_ronch += image
+            
+            hdf.flush()
+        
+        #print('{}, {}'.format(np.max(mean_ronch), np.min(mean_ronch)))
+        
+        h5_ronch[:] = mean_ronch/num_files
+        
+        hdf.flush()
+        hdf.close()
+        
+        
+    def __getfilelist(self, path, ftype='all'):
+        """
+        Returns a list of all files in the directory given by path
+        
+        Parameters
+        ---------------
+        path : string / unicode
+            absolute path to directory containing files
+        ftype : this file types to return in file_list. (optional. Default is all) 
+        
+        Returns
+        ----------
+        file_list : list of strings
+            names of all files in directory located at path
+        numfiles : unsigned int
+            number of files in file_list
+        """    
+        
+
+        # Get all files in directory
+        file_list = os.listdir(path)
+        
+        # If no file type specified, return full list
+        if ftype=='all':
+            numfiles = len(file_list)
+            return (file_list, numfiles)
+
+        # Remove files of type other than the request ftype from the list
+        new_file_list = []
+        for this_thing in file_list:
+            split = os.path.splitext(this_thing)
+            if len(split) <2:
+                continue
+            ext = split[1]
+            if ext == ftype:
+                new_file_list.append(this_thing)
+
+        numfiles = len(new_file_list)
+        
+        return (new_file_list,numfiles)
+