be_odf.py 44.8 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:24:12 2015

@author: Suhas Somnath, Stephen Jesse
"""

Somnath, Suhas's avatar
Somnath, Suhas committed
8
from __future__ import division  # int/int = float
9

Somnath, Suhas's avatar
Somnath, Suhas committed
10
from os import path, listdir, remove
11
from warnings import warn
Somnath, Suhas's avatar
Somnath, Suhas committed
12
13
import numpy as np
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
14
15

from .be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, createSpecVals
Somnath, Suhas's avatar
Somnath, Suhas committed
16
from .translator import Translator
Somnath, Suhas's avatar
Somnath, Suhas committed
17
from .utils import makePositionMat, getPositionSlicing, generateDummyMainParms
Somnath, Suhas's avatar
Somnath, Suhas committed
18
19
20
from ..hdf_utils import getH5DsetRefs, linkRefs, calc_chunks
from ..io_hdf5 import ioHDF5
from ..microdata import MicroDataGroup, MicroDataset
21

22
23
nf32 = np.dtype([('super_band', np.float32), ('inter_bin_band', np.float32),
                 ('sub_band', np.float32)])
24

Somnath, Suhas's avatar
Somnath, Suhas committed
25
26
27
28
29
class BEodfTranslator(Translator):
    """
    Translates either the Band Excitation (BE) scan or Band Excitation 
    Polarization Switching (BEPS) data format from the old data format(s) to .h5
    """
Chris Smith's avatar
Chris Smith committed
30
31
32
33
34
35
    def __init__(self, *args, **kwargs):
        super(BEodfTranslator, self).__init__(*args, **kwargs)

        self.hdf = None
        self.h5_raw = None

Somnath, Suhas's avatar
Somnath, Suhas committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False):
        """
        Translates .dat data file(s) to a single .h5 file
        
        Parameters
        -------------
        file_path : String / Unicode
            Absolute file path for one of the data files. 
            It is assumed that this file is of the OLD data format.
        show_plots : (optional) Boolean
            Whether or not to show intermediate plots
        save_plots : (optional) Boolean
            Whether or not to save plots to disk
        do_histogram : (optional) Boolean
            Whether or not to construct histograms to visualize data quality. Note - this takes a fair amount of time
            
        Returns
        ----------
        h5_path : String / Unicode
            Absolute path of the resultant .h5 file
        """
        (folder_path, basename) = path.split(file_path)
Chris Smith's avatar
Chris Smith committed
58
        (basename, path_dict) = self._parsefilepath(file_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
59
            
Somnath, Suhas's avatar
Somnath, Suhas committed
60
        h5_path = path.join(folder_path, basename + '.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
61
62
63
64
65
66
67
        tot_bins_multiplier = 1
        udvs_denom = 2
        
        if 'parm_txt' in path_dict.keys():
            (isBEPS,parm_dict) = parmsToDict(path_dict['parm_txt'])
        elif 'old_mat_parms' in path_dict.keys():
            isBEPS = True
Somnath, Suhas's avatar
Somnath, Suhas committed
68
            parm_dict = self.__get_parms_from_old_mat(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
        else:
            warn('No parameters file found! Cannot translate this dataset!')
            return
          
        ignored_plt_grps = []
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'
            
            field_mode = parm_dict['VS_measure_in_field_loops']
            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'
            
            if not std_expt:
                warn('This translator does not handle user defined voltage spectroscopy')
                return
            
            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode']) 
            
Somnath, Suhas's avatar
Somnath, Suhas committed
86
            if parm_dict['VS_mode'] in ['DC modulation mode', 'current mode']:
Somnath, Suhas's avatar
Somnath, Suhas committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
                if field_mode == 'in and out-of-field':
                    tot_bins_multiplier = 2
                    udvs_denom = 1
                else:
                    if field_mode == 'out-of-field':
                        ignored_plt_grps = ['in-field']
                    else:
                        ignored_plt_grps = ['out-of-field']
            else:
                tot_bins_multiplier = 1
                udvs_denom = 1
                    
        else:
            spec_label = 'None'
            parm_dict['data_type'] = 'BELineData'
            
        # Check file sizes:
        if 'read_real' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
105
106
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
Somnath, Suhas's avatar
Somnath, Suhas committed
107
108
109
110
111
112
113
114
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])
            
        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

        add_pix = False         
Somnath, Suhas's avatar
Somnath, Suhas committed
115
116
117
118
        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
        num_pix = num_rows*num_cols
        tot_bins = real_size/(num_pix*4)
Chris Smith's avatar
Chris Smith committed
119
120
        # Check for case where only a single pixel is missing.
        check_bins = real_size/((num_pix-1)*4)
Somnath, Suhas's avatar
Somnath, Suhas committed
121
122
123
        
        if tot_bins % 1 and check_bins % 1: 
            warn('Aborting! Some parameter appears to have changed in-between')
Somnath, Suhas's avatar
Somnath, Suhas committed
124
            return
Somnath, Suhas's avatar
Somnath, Suhas committed
125
        elif not tot_bins % 1:
Chris Smith's avatar
Chris Smith committed
126
            # Everything's ok
Somnath, Suhas's avatar
Somnath, Suhas committed
127
128
129
130
131
132
133
134
135
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.') 
            add_pix = True 
         
        tot_bins = int(tot_bins)*tot_bins_multiplier
        
        if 'parm_mat' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
136
            (bin_inds, bin_freqs, bin_FFT, ex_wfm) = self.__read_parms_mat(path_dict['parm_mat'], isBEPS)
Somnath, Suhas's avatar
Somnath, Suhas committed
137
        elif 'old_mat_parms' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
138
            (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__read_old_mat_be_vecs(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        else:
            band_width = parm_dict['BE_band_width_[Hz]']*(0.5 - parm_dict['BE_band_edge_trim'])
            st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
            en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width            
            bin_freqs = np.linspace(st_f, en_f, tot_bins, dtype=np.float32)
            
            print('No parms .mat file found.... Filling dummy values into ancillary datasets.')
            bin_inds = np.zeros(shape=tot_bins, dtype=np.int32)
            bin_FFT = np.zeros(shape=tot_bins, dtype=np.complex64)
            ex_wfm = np.zeros(shape=100, dtype=np.float32)
            
        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)
            
        self.FFT_BE_wave = bin_FFT
        pos_mat = makePositionMat([num_cols, num_rows])
Somnath, Suhas's avatar
Somnath, Suhas committed
158
        pos_slices = getPositionSlicing(['X', 'Y'], num_pix)
Somnath, Suhas's avatar
Somnath, Suhas committed
159
160
161
162
        
        ds_ex_wfm = MicroDataset('Excitation_Waveform', ex_wfm)     
        ds_pos_ind = MicroDataset('Position_Indices', pos_mat, dtype=np.uint32)
        ds_pos_ind.attrs['labels'] = pos_slices
Somnath, Suhas's avatar
Somnath, Suhas committed
163
        ds_pos_ind.attrs['units'] = ['' for _ in range(len(pos_slices))]
Somnath, Suhas's avatar
Somnath, Suhas committed
164
165
        ds_pos_val = MicroDataset('Position_Values', np.float32(pos_mat))
        ds_pos_val.attrs['labels'] = pos_slices
Somnath, Suhas's avatar
Somnath, Suhas committed
166
        ds_pos_val.attrs['units'] = ['' for _ in range(len(pos_slices))]
Somnath, Suhas's avatar
Somnath, Suhas committed
167
168
169
#         ds_pos_labs = MicroDataset('Position_Labels',np.array(pos_labs))
        
        if isBEPS:
Somnath, Suhas's avatar
Somnath, Suhas committed
170
            (UDVS_labs, UDVS_units, UDVS_mat) = self.__build_udvs_table(parm_dict)
Somnath, Suhas's avatar
Somnath, Suhas committed
171
172
173
174
           
#             Remove the unused plot group columns before proceeding:
            (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
           
Somnath, Suhas's avatar
Somnath, Suhas committed
175
            spec_inds = np.zeros(shape=(2, tot_bins), dtype=np.uint)
Somnath, Suhas's avatar
Somnath, Suhas committed
176
177
178
179
180
181
182
                      
#             Will assume that all excitation waveforms have same number of bins
            num_actual_udvs_steps = UDVS_mat.shape[0]/udvs_denom
            bins_per_step = tot_bins/num_actual_udvs_steps
           
            if bins_per_step % 1:
                warn('Non integer number of bins per step!')
Somnath, Suhas's avatar
Somnath, Suhas committed
183
184
                print('UDVS mat shape: {}, total bins: {}, bins per step: {}'.format(UDVS_mat.shape, tot_bins,
                                                                                     bins_per_step))
Somnath, Suhas's avatar
Somnath, Suhas committed
185
186
187
188
189
190
                return
            
            bins_per_step = int(bins_per_step)
            num_actual_udvs_steps = int(num_actual_udvs_steps)
               
            stind = 0           
Somnath, Suhas's avatar
Somnath, Suhas committed
191
192
193
194
195
            for step_index in range(UDVS_mat.shape[0]):  
                if UDVS_mat[step_index, 2] < 1E-3: # invalid AC amplitude
                    continue  # skip
                spec_inds[0, stind:stind+bins_per_step] = np.arange(bins_per_step, dtype=np.uint32) # Bin step
                spec_inds[1, stind:stind+bins_per_step] = step_index * np.ones(bins_per_step, dtype=np.uint32) # UDVS step
Somnath, Suhas's avatar
Somnath, Suhas committed
196
                stind += bins_per_step
Somnath, Suhas's avatar
Somnath, Suhas committed
197
            del stind, step_index
Somnath, Suhas's avatar
Somnath, Suhas committed
198
           
Somnath, Suhas's avatar
Somnath, Suhas committed
199
        else:  # BE Line
Somnath, Suhas's avatar
Somnath, Suhas committed
200
            self.signal_type = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
201
            self.expt_type = 1  # Stephen has not used this index for some reason
Somnath, Suhas's avatar
Somnath, Suhas committed
202
203
            num_actual_udvs_steps = 1
            bins_per_step = tot_bins
Somnath, Suhas's avatar
Somnath, Suhas committed
204
            UDVS_labs = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'be-line']
Somnath, Suhas's avatar
Somnath, Suhas committed
205
            UDVS_units = ['', 'V', 'A', '', '', '']
Somnath, Suhas's avatar
Somnath, Suhas committed
206
207
            UDVS_mat = np.array([1, 0, parm_dict['BE_amplitude_[V]'], 1, 1, 1],
                                dtype=np.float32).reshape(1, len(UDVS_labs))
Somnath, Suhas's avatar
Somnath, Suhas committed
208
209
210
211
212
213
214
215

            spec_inds = np.vstack((np.arange(tot_bins, dtype=np.uint), np.zeros(tot_bins, dtype=np.uint32)))
        
        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
        parm_dict['num_UDVS_steps'] = num_actual_udvs_steps 
        
Somnath, Suhas's avatar
Somnath, Suhas committed
216
        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
217
        for col_ind, col_name in enumerate(UDVS_labs):
Somnath, Suhas's avatar
Somnath, Suhas committed
218
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind+1))
Somnath, Suhas's avatar
Somnath, Suhas committed
219
220
221
222
223
224
225
        ds_UDVS = MicroDataset('UDVS', UDVS_mat)
        ds_UDVS.attrs['labels'] = udvs_slices
        ds_UDVS.attrs['units'] = UDVS_units
#         ds_udvs_labs = MicroDataset('UDVS_Labels',np.array(UDVS_labs))
        ds_UDVS_inds = MicroDataset('UDVS_Indices', spec_inds[1])        
        
#         ds_spec_labs = MicroDataset('Spectroscopic_Labels',np.array(['Bin','UDVS_Step']))
Somnath, Suhas's avatar
Somnath, Suhas committed
226
        ds_bin_steps = MicroDataset('Bin_Step', np.arange(bins_per_step, dtype=np.uint32), dtype=np.uint32)
Somnath, Suhas's avatar
Somnath, Suhas committed
227
228
229
230
231
232
        
        # Need to add the Bin Waveform type - infer from UDVS        
        exec_bin_vec = self.signal_type*np.ones(len(bin_inds), dtype=np.int32)

        if self.expt_type == 2:
            # Need to double the vectors:
Somnath, Suhas's avatar
Somnath, Suhas committed
233
234
235
            exec_bin_vec = np.hstack((exec_bin_vec, -1*exec_bin_vec))
            bin_inds = np.hstack((bin_inds, bin_inds))
            bin_freqs = np.hstack((bin_freqs, bin_freqs))
Somnath, Suhas's avatar
Somnath, Suhas committed
236
            # This is wrong but I don't know what else to do
Somnath, Suhas's avatar
Somnath, Suhas committed
237
            bin_FFT = np.hstack((bin_FFT, bin_FFT))
Somnath, Suhas's avatar
Somnath, Suhas committed
238
239
240
241
242
243
244
        
        ds_bin_inds = MicroDataset('Bin_Indices', bin_inds, dtype=np.uint32)       
        ds_bin_freq = MicroDataset('Bin_Frequencies', bin_freqs)        
        ds_bin_FFT = MicroDataset('Bin_FFT', bin_FFT)
        ds_wfm_typ = MicroDataset('Bin_Wfm_Type', exec_bin_vec)
        
        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
Somnath, Suhas's avatar
Somnath, Suhas committed
245
246
247
248
249
250
251
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_labs_names = createSpecVals(UDVS_mat,
                                                                                                     spec_inds,
                                                                                                     bin_freqs,
                                                                                                     exec_bin_vec,
                                                                                                     parm_dict,
                                                                                                     UDVS_labs,
                                                                                                     UDVS_units)
Chris Smith's avatar
Chris Smith committed
252

Somnath, Suhas's avatar
Somnath, Suhas committed
253
254
255
256
257
258
        spec_vals_slices = dict()
#         if len(spec_vals_labs) == 1:
#             spec_vals_slices[spec_vals_labs[0]]=(slice(0,1,None),)
#         else:

        for row_ind, row_name in enumerate(spec_vals_labs):
Somnath, Suhas's avatar
Somnath, Suhas committed
259
            spec_vals_slices[row_name] = (slice(row_ind, row_ind+1), slice(None))
Somnath, Suhas's avatar
Somnath, Suhas committed
260
261
262
263

        ds_spec_mat = MicroDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
        ds_spec_mat.attrs['labels'] = spec_vals_slices
        ds_spec_mat.attrs['units'] = spec_vals_units                   
Somnath, Suhas's avatar
Somnath, Suhas committed
264
        ds_spec_vals_mat = MicroDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
Somnath, Suhas's avatar
Somnath, Suhas committed
265
266
267
        ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
        ds_spec_vals_mat.attrs['units'] = spec_vals_units
        for entry in spec_vals_labs_names:
Chris Smith's avatar
Chris Smith committed
268
            label = entry[0]+'_parameters'
Somnath, Suhas's avatar
Somnath, Suhas committed
269
            names = entry[1]
Somnath, Suhas's avatar
Somnath, Suhas committed
270
271
            ds_spec_mat.attrs[label] = names
            ds_spec_vals_mat.attrs[label] = names
Chris Smith's avatar
Chris Smith committed
272

Somnath, Suhas's avatar
Somnath, Suhas committed
273
        # Noise floor should be of shape: (udvs_steps x 3 x positions)
Somnath, Suhas's avatar
Somnath, Suhas committed
274
275
        ds_noise_floor = MicroDataset('Noise_Floor', np.zeros(shape=(num_pix, num_actual_udvs_steps), dtype=nf32),
                                      chunking=(1, num_actual_udvs_steps))
Somnath, Suhas's avatar
Somnath, Suhas committed
276
277

        """
Chris Smith's avatar
Chris Smith committed
278
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares
Somnath, Suhas's avatar
Somnath, Suhas committed
279
        of UDVS steps by pixels.  N is determined dynamically based on the dimensions of the
Chris Smith's avatar
Chris Smith committed
280
        dataset.  Currently it is set such that individual chunks are less than 10kB in size.
Somnath, Suhas's avatar
Somnath, Suhas committed
281
282
283
        
        Chris Smith -- csmith55@utk.edu
        """
Chris Smith's avatar
Chris Smith committed
284
285
286
287
288
289
290
291
        BEPS_chunks = calc_chunks([num_pix, tot_bins],
                                  np.complex64(0).itemsize,
                                  unit_chunks=(1, bins_per_step))
        ds_main_data = MicroDataset('Raw_Data', data=[],
                                    maxshape=(num_pix, tot_bins),
                                    dtype=np.complex64,
                                    chunking=BEPS_chunks,
                                    compression='gzip')
Somnath, Suhas's avatar
Somnath, Suhas committed
292
293
294
295
296
        
        chan_grp = MicroDataGroup('Channel_')
        chan_grp.attrs['Channel_Input'] = parm_dict['IO_Analog_Input_1']
        chan_grp.addChildren([ds_main_data, ds_noise_floor])
        chan_grp.addChildren([ds_ex_wfm, ds_pos_ind, ds_pos_val, ds_spec_mat, ds_UDVS,
Chris Smith's avatar
Chris Smith committed
297
298
                              ds_bin_steps, ds_bin_inds, ds_bin_freq, ds_bin_FFT,
                              ds_wfm_typ, ds_spec_vals_mat, ds_UDVS_inds])
Somnath, Suhas's avatar
Somnath, Suhas committed
299
300
301
302
303
304
305
306
        
        # technically should change the date, etc.
        meas_grp = MicroDataGroup('Measurement_')
        meas_grp.attrs = parm_dict
        meas_grp.addChildren([chan_grp])
        
        spm_data = MicroDataGroup('')
        global_parms = generateDummyMainParms()
Somnath, Suhas's avatar
Somnath, Suhas committed
307
308
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
309
310
311
312
        try:
            global_parms['experiment_date'] = parm_dict['File_date_and_time']
        except KeyError:
            global_parms['experiment_date'] = '1:1:1'
Chris Smith's avatar
Chris Smith committed
313

Somnath, Suhas's avatar
Somnath, Suhas committed
314
        # assuming that the experiment was completed:
Somnath, Suhas's avatar
Somnath, Suhas committed
315
316
317
        global_parms['current_position_x'] = parm_dict['grid_num_cols']-1
        global_parms['current_position_y'] = parm_dict['grid_num_rows']-1
        global_parms['data_type'] = parm_dict['data_type']
Somnath, Suhas's avatar
Somnath, Suhas committed
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        global_parms['translator'] = 'ODF'
            
        spm_data.attrs = global_parms
        spm_data.addChildren([meas_grp])
        
        if path.exists(h5_path):
            remove(h5_path)
        
        # Write everything except for the main data.
        self.hdf = ioHDF5(h5_path)
        
        h5_refs = self.hdf.writeData(spm_data)
                    
        self.h5_raw = getH5DsetRefs(['Raw_Data'], h5_refs)[0]
            
Somnath, Suhas's avatar
Somnath, Suhas committed
333
334
335
336
        # Now doing linkrefs:
        aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values',
                        'Spectroscopic_Indices', 'UDVS', 'Bin_Step', 'Bin_Indices', 'UDVS_Indices',
                        'Bin_Frequencies', 'Bin_FFT', 'Bin_Wfm_Type', 'Noise_Floor', 'Spectroscopic_Values']
337
        linkRefs(self.h5_raw, getH5DsetRefs(aux_ds_names, h5_refs))
Somnath, Suhas's avatar
Somnath, Suhas committed
338

Chris Smith's avatar
Chris Smith committed
339
        self._read_data(UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix)
Somnath, Suhas's avatar
Somnath, Suhas committed
340
341
342
        
        generatePlotGroups(self.h5_raw, self.hdf, self.mean_resp, folder_path, basename,
                           self.max_resp, self.min_resp, max_mem_mb=self.max_ram,
Somnath, Suhas's avatar
Somnath, Suhas committed
343
                           spec_label=spec_label, show_plots=show_plots, save_plots=save_plots,
Somnath, Suhas's avatar
Somnath, Suhas committed
344
345
346
347
348
                           do_histogram=do_histogram)
        
        self.hdf.close()
        
        return h5_path
Chris Smith's avatar
Chris Smith committed
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376

    def _read_data(self, UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix):
        """
        Checks if the data is BEPS or BELine and calls the correct function to read the data from
        file

        Parameters
        ----------
        UDVS_mat : numpy.ndarray of float
            UDVS table
        parm_dict : dict
            Experimental parameters
        path_dict : dict
            Dictionary of data files to be read
        real_size : dict
            Size of each data file
        isBEPS : boolean
            Is the data BEPS
        add_pix : boolean
            Does the reader need to add extra pixels to the end of the dataset

        Returns
        -------
        None
        """
        # Now read the raw data files:
        if not isBEPS:
            # Do this for all BE-Line (always small enough to read in one shot)
Somnath, Suhas's avatar
Somnath, Suhas committed
377
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'])
Chris Smith's avatar
Chris Smith committed
378
379
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'out-of-field':
            # Do this for out-of-field BEPS ONLY that is also small (256 MB)
Somnath, Suhas's avatar
Somnath, Suhas committed
380
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'])
Chris Smith's avatar
Chris Smith committed
381
382
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'in-field':
            # Do this for in-field only
Somnath, Suhas's avatar
Somnath, Suhas committed
383
            self.__quick_read_data(path_dict['write_real'], path_dict['write_imag'])
Chris Smith's avatar
Chris Smith committed
384
385
        else:
            # Large BEPS datasets OR those with in-and-out of field
Somnath, Suhas's avatar
Somnath, Suhas committed
386
            self.__read_beps_data(path_dict, UDVS_mat.shape[0], parm_dict['VS_measure_in_field_loops'], add_pix)
Chris Smith's avatar
Chris Smith committed
387
388
        self.hdf.file.flush()

Somnath, Suhas's avatar
Somnath, Suhas committed
389
    def __read_beps_data(self, path_dict, udvs_steps, mode, add_pixel=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
        """
        Reads the imaginary and real data files pixelwise and writes to the H5 file 
        
        Parameters 
        --------------------
        path_dict : dictionary
            Dictionary containing the absolute paths of the real and imaginary data files
        udvs_steps : unsigned int
            Number of UDVS steps
        mode : String / Unicode
            'in-field', 'out-of-field', or 'in and out-of-field'
        add_pixel : boolean. (Optional; default is False)
            If an empty pixel worth of data should be written to the end             
        
        Returns 
        -------------------- 
        None
        """
        
        print('---- reading pixel-by-pixel ----------')
        
        bytes_per_pix = self.h5_raw.shape[1]*4 
        step_size = self.h5_raw.shape[1]/udvs_steps          
        
        if mode == 'out-of-field':
Somnath, Suhas's avatar
Somnath, Suhas committed
415
416
            parsers = [BEodfParser(path_dict['read_real'], path_dict['read_imag'], 
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
417
        elif mode == 'in-field':
Somnath, Suhas's avatar
Somnath, Suhas committed
418
419
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'], 
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
420
421
422
423
424
425
426
        elif mode == 'in and out-of-field':
            # each file will only have half the udvs steps:
            if 0.5*udvs_steps % 1:
                warn('Odd number of UDVS')
                return
            udvs_steps = int(0.5*udvs_steps)
            # be careful - each pair contains only half the necessary bins - so read half
Somnath, Suhas's avatar
Somnath, Suhas committed
427
428
429
430
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'], 
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2)),
                       BEodfParser(path_dict['read_real'], path_dict['read_imag'], 
                                   self.h5_raw.shape[0], int(bytes_per_pix/2))]
Somnath, Suhas's avatar
Somnath, Suhas committed
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
            
            if step_size % 1:
                warn('weird number of bins per UDVS step. Exiting')
                return
            step_size = int(step_size)
        
        self.mean_resp = np.zeros(shape=(self.h5_raw.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)

        numpix = self.h5_raw.shape[0] 
        """ 
        Don't try to do the last step if a pixel is missing.   
        This will be handled after the loop. 
        """ 
        if add_pixel: 
Somnath, Suhas's avatar
Somnath, Suhas committed
447
            numpix -= 1 
Somnath, Suhas's avatar
Somnath, Suhas committed
448
        
Somnath, Suhas's avatar
Somnath, Suhas committed
449
        for pix_indx in range(numpix):
Somnath, Suhas's avatar
Somnath, Suhas committed
450
451
            if self.h5_raw.shape[0] > 5:
                if pix_indx % int(round(self.h5_raw.shape[0]/10)) == 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
452
                    print('Reading... {} complete'.format(round(100*pix_indx / self.h5_raw.shape[0])))
Somnath, Suhas's avatar
Somnath, Suhas committed
453
454
455
456
                    
            # get the raw stream from each parser
            pxl_data = list()
            for prsr in parsers:
Somnath, Suhas's avatar
Somnath, Suhas committed
457
                pxl_data.append(prsr.read_pixel())
Somnath, Suhas's avatar
Somnath, Suhas committed
458
459
460
461
462
463
464
            
            # interleave if both in and out of field
            # we are ignoring user defined possibilities...
            if mode == 'in and out-of-field':
                in_fld = pxl_data[0]
                out_fld = pxl_data[1]
                
Somnath, Suhas's avatar
Somnath, Suhas committed
465
466
467
468
469
                in_fld_2 = in_fld.reshape(udvs_steps, step_size)
                out_fld_2 = out_fld.reshape(udvs_steps, step_size)
                raw_mat = np.empty((udvs_steps*2, step_size), dtype=out_fld.dtype)
                raw_mat[0::2, :] = in_fld_2
                raw_mat[1::2, :] = out_fld_2
Somnath, Suhas's avatar
Somnath, Suhas committed
470
471
                raw_vec = raw_mat.reshape(in_fld.size + out_fld.size).transpose()
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
472
                raw_vec = pxl_data[0]  # only one parser
Somnath, Suhas's avatar
Somnath, Suhas committed
473
474
            self.max_resp[pix_indx] = np.max(np.abs(raw_vec))
            self.min_resp[pix_indx] = np.min(np.abs(raw_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
475
            self.mean_resp = (1/(pix_indx+1))*(raw_vec + pix_indx * self.mean_resp)
Somnath, Suhas's avatar
Somnath, Suhas committed
476
            
Somnath, Suhas's avatar
Somnath, Suhas committed
477
            self.h5_raw[pix_indx, :] = raw_vec[:]
Somnath, Suhas's avatar
Somnath, Suhas committed
478
479
480
481
            self.hdf.file.flush()
            
        # Add zeros to main_data for the missing pixel. 
        if add_pixel: 
Somnath, Suhas's avatar
Somnath, Suhas committed
482
            self.h5_raw[-1, :] = 0+0j             
Somnath, Suhas's avatar
Somnath, Suhas committed
483
484
485
            
        print('---- Finished reading files -----')
    
Somnath, Suhas's avatar
Somnath, Suhas committed
486
    def __quick_read_data(self, real_path, imag_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
487
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
488
489
490
491
492
493
494
495
        Returns information about the excitation BE waveform present in the .mat file

        Parameters
        -----------
        real_path : String / Unicode
            Absolute file path of the real data file
        imag_path : String / Unicode
            Absolute file path of the real data file
Somnath, Suhas's avatar
Somnath, Suhas committed
496
497
498
499
        """
        print('---- reading all data at once ----------')  

        parser = BEodfParser(real_path, imag_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
500
        raw_vec = parser.read_all_data()
Somnath, Suhas's avatar
Somnath, Suhas committed
501
                                      
Somnath, Suhas's avatar
Somnath, Suhas committed
502
        raw_mat = raw_vec.reshape(self.h5_raw.shape[0], self.h5_raw.shape[1])
Somnath, Suhas's avatar
Somnath, Suhas committed
503
504
                
        # Write to the h5 dataset:
Somnath, Suhas's avatar
Somnath, Suhas committed
505
506
507
508
        self.mean_resp = np.mean(raw_mat, axis=0)
        self.max_resp = np.amax(np.abs(raw_mat), axis=0)
        self.min_resp = np.amin(np.abs(raw_mat), axis=0)
        self.h5_raw[:, :] = raw_mat
Somnath, Suhas's avatar
Somnath, Suhas committed
509
510
511
512
        self.hdf.file.flush()

        print('---- Finished reading files -----')       
        
Chris Smith's avatar
Chris Smith committed
513
    def _parsefilepath(self, data_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
514
515
516
517
518
519
520
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary
        
        Parameters 
        --------------------
        data_filepath: String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
521
            Absolute path of any file in the same directory as the .dat files
Somnath, Suhas's avatar
Somnath, Suhas committed
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
        
        Returns 
        --------------------
        basename : String / Unicode
            Basename of the dataset      
        path_dict : Dictionary
            Dictionary containing absolute paths of all necessary data and parameter files
        """
        (folder_path, basename) = path.split(data_filepath)
        (super_folder, basename) = path.split(folder_path) 

        if basename.endswith('_d'):
            # Old old data format where the folder ended with a _d for some reason
            basename = basename[:-2]
        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()
        
        for file_name in listdir(folder_path):
Chris Smith's avatar
Chris Smith committed
545
            abs_path = path.join(folder_path, file_name)
Somnath, Suhas's avatar
Somnath, Suhas committed
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
            if file_name.endswith('.txt') and file_name.find('parm') > 0:
                path_dict['parm_txt'] = abs_path
            elif file_name.find('.mat') > 0:
                if file_name.find('more_parms') > 0:
                    path_dict['parm_mat'] = abs_path
                elif file_name == (basename + '.mat'):                   
                    path_dict['old_mat_parms'] = abs_path
            elif file_name.endswith('.dat'):
                # Need to account for the second AI channel here
                file_tag = 'read'
                if file_name.find('write') > 0:
                    file_tag = 'write'
                if file_name.find('real') > 0:
                    file_tag += '_real'
                elif file_name.find('imag') > 0:
                    file_tag += '_imag'
                path_dict[file_tag] = abs_path

Chris Smith's avatar
Chris Smith committed
564
        return basename, path_dict
Somnath, Suhas's avatar
Somnath, Suhas committed
565
566
567

    @staticmethod
    def __read_old_mat_be_vecs(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
        """
        Returns information about the excitation BE waveform present in the 
        more parms.mat file
        
        Parameters 
        --------------------
        filepath : String or unicode
            Absolute filepath of the .mat parameter file
        
        Returns 
        --------------------
        bin_inds : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        bin_w : 1D numpy float array
            Excitation bin Frequencies
        bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        BE_wave : 1D numpy float array
            Band Excitation waveform
        dc_amp_vec_full : 1D numpy float array
            spectroscopic waveform. 
            This information will be necessary for fixing the UDVS for AC modulation for example
        """
        matread = loadmat(file_path, squeeze_me=True)    
        BE_wave = matread['BE_wave']
Somnath, Suhas's avatar
Somnath, Suhas committed
593
        bin_inds = matread['bin_ind'] -1  # Python base 0
Somnath, Suhas's avatar
Somnath, Suhas committed
594
595
596
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
Somnath, Suhas's avatar
Somnath, Suhas committed
597
598
        bin_FFT = np.conjugate(FFT_full[bin_inds])
        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full
Somnath, Suhas's avatar
Somnath, Suhas committed
599
        
Somnath, Suhas's avatar
Somnath, Suhas committed
600
601
    @staticmethod
    def __get_parms_from_old_mat(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it
        
        Parameters 
        --------------------
        file_path : Unicode / String
            absolute filepath of the .mat file containing the parameters
            
        Returns 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)
        
        parm_dict['IO_rate'] = str(int(matread['AO_rate']/1E+6)) + ' MHz'
        
        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = position_vec[2]
        parm_dict['grid_num_cols'] = position_vec[3]
        
        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
Somnath, Suhas's avatar
Somnath, Suhas committed
629
            parm_dict['grid_num_rows'] = position_vec[0]  # set to number of present cols and rows
Somnath, Suhas's avatar
Somnath, Suhas committed
630
631
632
633
634
635
636
637
638
639
640
            parm_dict['grid_num_cols'] = position_vec[1]
    
        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
641
642
        parm_dict['BE_band_edge_smoothing_[s]'] = BE_parm_vec_1[4]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
Somnath, Suhas's avatar
Somnath, Suhas committed
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
        parm_dict['BE_window_adjustment'] = BE_parm_vec_1[6] 
        parm_dict['BE_points_per_step'] = 2**int(BE_parm_vec_1[7])
        parm_dict['BE_repeats'] = 2**int(BE_parm_vec_1[8])
        try:
            parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        except KeyError:
            parm_dict['BE_bins_per_read'] = len(matread['bin_w'])
    
        assembly_parm_vec = matread['assembly_parm_vec']
        
        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
        
        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'
            
Somnath, Suhas's avatar
Somnath, Suhas committed
666
667
        # num_driving_bands = assembly_parm_vec[0]  # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1]  # 0 parallel 1 series
Somnath, Suhas's avatar
Somnath, Suhas committed
668
669
670
671
672
673
674
        
        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']
           
        VS_start_V = VS_parms[4] 
        VS_start_loop_amp = VS_parms[5] 
        VS_final_loop_amp = VS_parms[6] 
Somnath, Suhas's avatar
Somnath, Suhas committed
675
        # VS_read_write_ratio = VS_parms[8]  # 1 <- SS_read_write_ratio
Somnath, Suhas's avatar
Somnath, Suhas committed
676
        
Somnath, Suhas's avatar
Somnath, Suhas committed
677
        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
678
679
680
681
682
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3] 
        parm_dict['VS_steps_per_full_cycle'] = VS_parms[7]
        parm_dict['VS_cycle_fraction'] = 'full'
        parm_dict['VS_cycle_phase_shift'] = 0 
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
Somnath, Suhas's avatar
Somnath, Suhas committed
683
684
685
686
687
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
688
689
690
        
        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
Somnath, Suhas's avatar
Somnath, Suhas committed
691
            parm_dict['VS_amplitude_[V]'] = 0.5*(max(dc_amp_vec_full) - min(dc_amp_vec_full))  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
692
693
694
695
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)     
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
Somnath, Suhas's avatar
Somnath, Suhas committed
696
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
697
698
            parm_dict['VS_offset_[V]'] = 0
            parm_dict['VS_number_of_cycles'] = 1                             
Somnath, Suhas's avatar
Somnath, Suhas committed
699
700
701
702
703
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
704
705
706
        elif VS_parms[0] == 2:
            # AC mode 
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
Somnath, Suhas's avatar
Somnath, Suhas committed
707
708
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
            parm_dict['VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
Somnath, Suhas's avatar
Somnath, Suhas committed
709
710
711
712
713
        else:
            parm_dict['VS_mode'] = 'Custom'
    
        return parm_dict
        
Somnath, Suhas's avatar
Somnath, Suhas committed
714
715
    @staticmethod
    def __read_parms_mat(file_path, is_beps):
Somnath, Suhas's avatar
Somnath, Suhas committed
716
717
718
719
720
        """
        Returns information about the excitation BE waveform present in the more parms.mat file
        
        Parameters 
        --------------------
Somnath, Suhas's avatar
Somnath, Suhas committed
721
        file_path : String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
722
            Absolute filepath of the .mat parameter file
Somnath, Suhas's avatar
Somnath, Suhas committed
723
        is_beps : Boolean
Somnath, Suhas's avatar
Somnath, Suhas committed
724
725
726
727
728
729
730
731
732
733
734
735
736
            Whether or not this is BEPS or BE-Line
        
        Returns 
        --------------------
        BE_bin_ind : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        BE_bin_w : 1D numpy float array
            Excitation bin Frequencies
        BE_bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        ex_wfm : 1D numpy float array
            Band Excitation waveform
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
737
        if not path.exists(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
738
739
            warn('BEodfTranslator - NO More parms file found')
            return None
Somnath, Suhas's avatar
Somnath, Suhas committed
740
        if is_beps:
Somnath, Suhas's avatar
Somnath, Suhas committed
741
742
743
            fft_name = 'FFT_BE_wave'
        else:
            fft_name = 'FFT_BE_rev_wave'
Somnath, Suhas's avatar
Somnath, Suhas committed
744
745
746
747
        matread = loadmat(file_path, variable_names=['BE_bin_ind', 'BE_bin_w', fft_name])
        BE_bin_ind = np.squeeze(matread['BE_bin_ind']) - 1   # From Matlab (base 1) to Python (base 0)
        BE_bin_w = np.squeeze(matread['BE_bin_w'])
        FFT_full = np.complex64(np.squeeze(matread[fft_name]))
Somnath, Suhas's avatar
Somnath, Suhas committed
748
        # For whatever weird reason, the sign of the imaginary portion is flipped. Correct it:
Somnath, Suhas's avatar
Somnath, Suhas committed
749
750
751
752
        #BE_bin_FFT = np.conjugate(FFT_full[BE_bin_ind])
        BE_bin_FFT = np.zeros(len(BE_bin_ind), dtype=np.complex64)
        BE_bin_FFT.real = np.real(FFT_full[BE_bin_ind])
        BE_bin_FFT.imag = -1*np.imag(FFT_full[BE_bin_ind])
Somnath, Suhas's avatar
Somnath, Suhas committed
753
754
        
        ex_wfm = np.real(np.fft.ifft(np.fft.ifftshift(FFT_full)))
Somnath, Suhas's avatar
Somnath, Suhas committed
755
756

        return BE_bin_ind, BE_bin_w, BE_bin_FFT, ex_wfm
Somnath, Suhas's avatar
Somnath, Suhas committed
757
        
Somnath, Suhas's avatar
Somnath, Suhas committed
758
    def __build_udvs_table(self, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
        """
        Generates the UDVS table using the parameters
        
        Parameters 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        
        Returns 
        --------------------      
        UD_VS_table_label : List of strings
            Labels for columns in the UDVS table
        UD_VS_table_unit : List of strings
            Units for the columns in the UDVS table
        UD_VS_table : 2D numpy float array
            UDVS data table
        """
    
Somnath, Suhas's avatar
Somnath, Suhas committed
777
        def translate_val(target, strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
778
779
            """
            Internal function - Interprets the provided value using the provided lookup table
Somnath, Suhas's avatar
Somnath, Suhas committed
780
781
782
783
784
785
786
787
788

            Parameters
            ----------
            target : String
                Item we are looking for in the strvals list
            strvals : list of strings
                List of source values
            numvals : list of numbers
                List of results
Somnath, Suhas's avatar
Somnath, Suhas committed
789
790
791
792
            """
        
            if len(strvals) is not len(numvals):
                return None    
Somnath, Suhas's avatar
Somnath, Suhas committed
793
            for strval, fltval in zip(strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
794
795
                if target == strval:
                    return fltval
Somnath, Suhas's avatar
Somnath, Suhas committed
796
            return None  # not found in list
Somnath, Suhas's avatar
Somnath, Suhas committed
797
798
            
        #% Extract values from parm text file    
Somnath, Suhas's avatar
Somnath, Suhas committed
799
        BE_signal_type = translate_val(parm_dict['BE_phase_content'], ['chirp-sinc hybrid','1/2 harmonic excitation','1/3 harmonic excitation','pure sine'],[1,2,3,4])
Somnath, Suhas's avatar
Somnath, Suhas committed
800
801
802
803
804
805
806
807
808
809
810
811
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type
        if BE_signal_type is 4:
            self.harmonic = 1
        BE_amp = parm_dict['BE_amplitude_[V]']
        
        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
        #VS_read_voltage = parm_dict['VS_read_voltage_[V]']
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
Somnath, Suhas's avatar
Somnath, Suhas committed
812
813
814
        VS_fraction = translate_val(parm_dict['VS_cycle_fraction'],
                                    ['full', '1/2', '1/4', '3/4'],
                                    [1., 0.5, 0.25, 0.75])
Somnath, Suhas's avatar
Somnath, Suhas committed
815
816
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
817
818
819
820
821
822
823
            VS_shift = translate_val(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
        VS_in_out_cond = translate_val(parm_dict['VS_measure_in_field_loops'],
                                       ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
        VS_ACDC_cond = translate_val(parm_dict['VS_mode'],
                                     ['DC modulation mode', 'AC modulation mode with time reversal',
                                      'load user defined VS Wave from file', 'current mode'],
                                     [0, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
824
825
826
827
828
829
830
831
832
833
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
        #FORC_repeats = parm_dict['# of FORC repeats']
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']
            
        #% build vector of voltage spectroscopy values
        
Somnath, Suhas's avatar
Somnath, Suhas committed
834
835
        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
            VS_amp_vec_1 = np.arange(0, 1+1/(VS_steps/4), 1/(VS_steps/4))
Somnath, Suhas's avatar
Somnath, Suhas committed
836
837
838
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
            VS_amp_vec_4 =  VS_amp_vec_1[1:-1]-1
Somnath, Suhas's avatar
Somnath, Suhas committed
839
840
841
842
843
844
845
846
847
848
849
850
851
            vs_amp_vec = VS_amp*(np.hstack((VS_amp_vec_1, VS_amp_vec_2,  VS_amp_vec_3, VS_amp_vec_4)))
            vs_amp_vec = np.roll(vs_amp_vec, int(np.floor(VS_steps/VS_fraction*VS_shift)))  # apply phase shift to VS wave
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps*VS_fraction))]  # cut VS waveform
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles)  # repeat VS waveform
            vs_amp_vec = vs_amp_vec+VS_offset
            
        elif VS_ACDC_cond == 2:  # AC voltage spectroscopy with time reversal
            vs_amp_vec = VS_amp * np.arange(1/(VS_steps/2/VS_fraction), 1 + 1/(VS_steps/2/VS_fraction),
                                            1/(VS_steps/2/VS_fraction))
            vs_amp_vec = np.roll(vs_amp_vec,
                                 int(np.floor(VS_steps/VS_fraction*VS_shift)))  # apply phase shift to VS wave
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps*VS_fraction/2))]  # cut VS waveform
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles * 2)  # repeat VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
852
853
            
        if FORC_cycles > 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
854
            vs_amp_vec = vs_amp_vec/np.max(np.abs(vs_amp_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
855
856
857
858
859
860
            FORC_cycle_vec = np.arange(0, FORC_cycles+1, FORC_cycles/(FORC_cycles-1))
            FORC_A_vec = FORC_cycle_vec*(FORC_A2-FORC_A1)/FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec*(FORC_B2-FORC_B1)/FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec-FORC_B_vec)/2
            FORC_off_vec = (FORC_A_vec+FORC_B_vec)/2
            
Somnath, Suhas's avatar
Somnath, Suhas committed
861
862
863
            VS_amp_mat = np.tile(vs_amp_vec, [FORC_cycles, 1])
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(vs_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(vs_amp_vec), 1]).transpose()
Somnath, Suhas's avatar
Somnath, Suhas committed
864
            VS_amp_mat = VS_amp_mat*FORC_amp_mat + FORC_off_mat
Somnath, Suhas's avatar
Somnath, Suhas committed
865
            vs_amp_vec = VS_amp_mat.reshape(int(FORC_cycles*VS_cycles*VS_fraction*VS_steps))
Somnath, Suhas's avatar
Somnath, Suhas committed
866
            
Somnath, Suhas's avatar
Somnath, Suhas committed
867
868
        # Build UDVS table:
        if VS_ACDC_cond is 0 or VS_ACDC_cond is 4:  # DC voltage spectroscopy or current mode
Somnath, Suhas's avatar
Somnath, Suhas committed
869
870
            
            if VS_ACDC_cond is 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
871
                UD_dc_vec = np.vstack((vs_amp_vec, np.zeros(len(vs_amp_vec))))
Somnath, Suhas's avatar
Somnath, Suhas committed
872
            if VS_ACDC_cond is 4:
Somnath, Suhas's avatar
Somnath, Suhas committed
873
                UD_dc_vec = np.vstack((vs_amp_vec, vs_amp_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
874
        
Somnath, Suhas's avatar
Somnath, Suhas committed
875
            UD_dc_vec = UD_dc_vec.transpose().reshape(UD_dc_vec.size)
Somnath, Suhas's avatar
Somnath, Suhas committed
876
877
            num_VS_steps = UD_dc_vec.size
                        
Somnath, Suhas's avatar
Somnath, Suhas committed
878
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'in-field', 'out-of-field']
Somnath, Suhas's avatar
Somnath, Suhas committed
879
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'V', 'V']
Somnath, Suhas's avatar
Somnath, Suhas committed
880
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Somnath, Suhas's avatar
Somnath, Suhas committed
881
            
Somnath, Suhas's avatar
Somnath, Suhas committed
882
883
            udvs_table[:, 0] = np.arange(0, num_VS_steps)  # Python base 0
            udvs_table[:, 1] = UD_dc_vec
Somnath, Suhas's avatar
Somnath, Suhas committed
884
            
Somnath, Suhas's avatar
Somnath, Suhas committed
885
886
            BE_IF_switch = np.abs(np.imag(np.exp(1j*np.pi/2*np.arange(1, num_VS_steps+1))))
            BE_OF_switch = np.abs(np.real(np.exp(1j*np.pi/2*np.arange(1, num_VS_steps+1))))
Somnath, Suhas's avatar
Somnath, Suhas committed
887
            
Somnath, Suhas's avatar
Somnath, Suhas committed
888
889
890
891
892
893
            if VS_in_out_cond is 0:  # out of field only
                udvs_table[:, 2] = BE_amp * BE_OF_switch
            elif VS_in_out_cond is 1:  # in field only
                udvs_table[:, 2] = BE_amp * BE_IF_switch
            elif VS_in_out_cond is 2:  # both in and out of field
                udvs_table[:, 2] = BE_amp * np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
894
            
Somnath, Suhas's avatar
Somnath, Suhas committed
895
896
            udvs_table[:, 3] = np.ones(num_VS_steps)  # wave type
            udvs_table[:, 4] = np.ones(num_VS_steps) * BE_signal_type  # wave mod
Somnath, Suhas's avatar
Somnath, Suhas committed
897
            
Somnath, Suhas's avatar
Somnath, Suhas committed
898
899
            udvs_table[:, 5] = float('NaN')*np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN')*np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
900
                            
Somnath, Suhas's avatar
Somnath, Suhas committed
901
902
            udvs_table[BE_IF_switch == 1, 5] = udvs_table[BE_IF_switch == 1, 1]
            udvs_table[BE_OF_switch == 1, 6] = udvs_table[BE_IF_switch == 1, 1]
Somnath, Suhas's avatar
Somnath, Suhas committed
903
            
Somnath, Suhas's avatar
Somnath, Suhas committed
904
        elif VS_ACDC_cond is 2:  # AC voltage spectroscopy
Somnath, Suhas's avatar
Somnath, Suhas committed
905
        
Somnath, Suhas's avatar
Somnath, Suhas committed
906
            num_VS_steps = vs_amp_vec.size
Somnath, Suhas's avatar
Somnath, Suhas committed
907
908
909
910
911
912
913
            half = int(0.5*num_VS_steps)
            
            if num_VS_steps is not half * 2:
                warn('Odd number of UDVS steps found. Exiting!')
                return
                
            UD_dc_vec = VS_offset*np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
914
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'forward', 'reverse']
Somnath, Suhas's avatar
Somnath, Suhas committed
915
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'A', 'A']
Somnath, Suhas's avatar
Somnath, Suhas committed
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
            udvs_table[:, 0] = np.arange(1, num_VS_steps+1)
            udvs_table[:, 1] = UD_dc_vec
            udvs_table[:, 2] = vs_amp_vec
            udvs_table[:, 3] = np.ones(num_VS_steps)
            udvs_table[:half, 4] = BE_signal_type*np.ones(half)
            udvs_table[half:, 4] = -1*BE_signal_type*np.ones(half)
            udvs_table[:, 5] = float('NaN')*np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN')*np.ones(num_VS_steps)
            udvs_table[:half, 5] = vs_amp_vec[:half]
            udvs_table[half:, 6] = vs_amp_vec[half:]
            
        return UD_VS_table_label, UD_VS_table_unit, udvs_table


Somnath, Suhas's avatar
Somnath, Suhas committed
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
class BEodfParser(object):
    """
    Objects that help in reading raw .dat files either a pixel at a time or all at once.
    """
    
    def __init__(self, real_path, imag_path, num_pix=None, bytes_per_pix=None):
        """
        This object reads the two binary data files (real and imaginary data).
        Use separate parser instances for in-field and out-field data sets.
        
        Parameters 
        --------------------
        real_path : String / Unicode
            absolute path of the binary file containing the real portion of the data
        imag_path : String / Unicode
            absolute path of the binary file containing the imaginary portion of the data
        num_pix : unsigned int
            Number of pixels in this image
        bytes_per_pix : unsigned int
            Number of bytes per pixel
        """
        self.f_real = open(real_path, "rb")
        self.f_imag = open(imag_path, "rb")               
        
        self.__num_pix__ = num_pix 
        self.__bytes_per_pix__ = bytes_per_pix
        self.__pix_indx__ = 0
            
Somnath, Suhas's avatar
Somnath, Suhas committed
959
    def read_pixel(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
960
961
        """
        Returns the content of the next pixel
Somnath, Suhas's avatar
Somnath, Suhas committed
962

Somnath, Suhas's avatar
Somnath, Suhas committed
963
964
965
966
967
968
969
970
971
972
        Returns 
        --------------------
        raw_vec : 1D numpy complex64 array
            Content of one pixel's data
        """
        
        if self.__pix_indx__ is self.__num_pix__:
            warn('BEodfParser - No more pixels to read!')
            return None
        
Somnath, Suhas's avatar
Somnath, Suhas committed
973
        self.f_real.seek(self.__pix_indx__*self.__bytes_per_pix__, 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
974
975
        real_vec = np.fromstring(self.f_real.read(self.__bytes_per_pix__), dtype='f')
        
Somnath, Suhas's avatar
Somnath, Suhas committed
976
        self.f_imag.seek(self.__pix_indx__*self.__bytes_per_pix__, 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
977
978
979
980
981
982
983
984
985
986
987
988
989
990
        imag_vec = np.fromstring(self.f_imag.read(self.__bytes_per_pix__), dtype='f')
        
        raw_vec = np.zeros(len(real_vec), dtype=np.complex64)
        raw_vec.real = real_vec
        raw_vec.imag = imag_vec
        
        self.__pix_indx__ += 1
        
        if self.__pix_indx__ is self.__num_pix__:
            self.f_real.close()
            self.f_imag.close()
        
        return raw_vec
        
Somnath, Suhas's avatar
Somnath, Suhas committed
991
    def read_all_data(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
992
993
        """
        Returns the complete contents of the file pair
Somnath, Suhas's avatar
Somnath, Suhas committed
994

Somnath, Suhas's avatar
Somnath, Suhas committed
995
996
997
998
999
        Returns 
        --------------------
        raw_vec : 1D numpy complex64 array
            Entire content of the file pair
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
1000
        self.f_real.seek(0, 0)