be_odf.py 47 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:24:12 2015

@author: Suhas Somnath, Stephen Jesse
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

Somnath, Suhas's avatar
Somnath, Suhas committed
10
from os import path, listdir, remove
11
from warnings import warn
12
import h5py
Somnath, Suhas's avatar
Somnath, Suhas committed
13
14
import numpy as np
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15

16
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
17
    createSpecVals, requires_conjugate, nf32
18
from ...core.io.translator import Translator, generate_dummy_main_parms
19
20
from ...core.io.write_utils import INDICES_DTYPE, VALUES_DTYPE, Dimension, calc_chunks
from ...core.io.hdf_utils import write_ind_val_dsets, write_main_dataset, write_region_references, \
21
    create_indexed_group, write_simple_attrs, write_book_keeping_attrs
22

23

Somnath, Suhas's avatar
Somnath, Suhas committed
24
25
26
27
28
class BEodfTranslator(Translator):
    """
    Translates either the Band Excitation (BE) scan or Band Excitation 
    Polarization Switching (BEPS) data format from the old data format(s) to .h5
    """
Unknown's avatar
Unknown committed
29

Chris Smith's avatar
Chris Smith committed
30
31
32
    def __init__(self, *args, **kwargs):
        super(BEodfTranslator, self).__init__(*args, **kwargs)
        self.h5_raw = None
33
        self.num_rand_spectra = kwargs.pop('num_rand_spectra', 1000)
Unknown's avatar
Unknown committed
34
35
36
        self.FFT_BE_wave = None
        self.signal_type = None
        self.expt_type = None
Chris Smith's avatar
Chris Smith committed
37

38
    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False, verbose=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
39
40
41
42
43
44
45
46
47
48
49
50
51
52
        """
        Translates .dat data file(s) to a single .h5 file
        
        Parameters
        -------------
        file_path : String / Unicode
            Absolute file path for one of the data files. 
            It is assumed that this file is of the OLD data format.
        show_plots : (optional) Boolean
            Whether or not to show intermediate plots
        save_plots : (optional) Boolean
            Whether or not to save plots to disk
        do_histogram : (optional) Boolean
            Whether or not to construct histograms to visualize data quality. Note - this takes a fair amount of time
53
54
        verbose : (optional) Boolean
            Whether or not to print statements
Somnath, Suhas's avatar
Somnath, Suhas committed
55
56
57
58
59
60
            
        Returns
        ----------
        h5_path : String / Unicode
            Absolute path of the resultant .h5 file
        """
61
        file_path = path.abspath(file_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
62
        (folder_path, basename) = path.split(file_path)
63
        (basename, path_dict) = self._parse_file_path(file_path)
Unknown's avatar
Unknown committed
64

Somnath, Suhas's avatar
Somnath, Suhas committed
65
        h5_path = path.join(folder_path, basename + '.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
66
67
        tot_bins_multiplier = 1
        udvs_denom = 2
Unknown's avatar
Unknown committed
68

Somnath, Suhas's avatar
Somnath, Suhas committed
69
        if 'parm_txt' in path_dict.keys():
Unknown's avatar
Unknown committed
70
            (isBEPS, parm_dict) = parmsToDict(path_dict['parm_txt'])
Somnath, Suhas's avatar
Somnath, Suhas committed
71
72
        elif 'old_mat_parms' in path_dict.keys():
            isBEPS = True
Somnath, Suhas's avatar
Somnath, Suhas committed
73
            parm_dict = self.__get_parms_from_old_mat(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
74
        else:
75
            raise IOError('No parameters file found! Cannot translate this dataset!')
Unknown's avatar
Unknown committed
76

Somnath, Suhas's avatar
Somnath, Suhas committed
77
78
79
        ignored_plt_grps = []
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'
Unknown's avatar
Unknown committed
80

Somnath, Suhas's avatar
Somnath, Suhas committed
81
82
            field_mode = parm_dict['VS_measure_in_field_loops']
            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'
Unknown's avatar
Unknown committed
83

Somnath, Suhas's avatar
Somnath, Suhas committed
84
            if not std_expt:
85
                raise ValueError('This translator does not handle user defined voltage spectroscopy')
Unknown's avatar
Unknown committed
86
87
88

            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode'])

Somnath, Suhas's avatar
Somnath, Suhas committed
89
            if parm_dict['VS_mode'] in ['DC modulation mode', 'current mode']:
Somnath, Suhas's avatar
Somnath, Suhas committed
90
91
92
93
94
95
96
97
98
99
100
                if field_mode == 'in and out-of-field':
                    tot_bins_multiplier = 2
                    udvs_denom = 1
                else:
                    if field_mode == 'out-of-field':
                        ignored_plt_grps = ['in-field']
                    else:
                        ignored_plt_grps = ['out-of-field']
            else:
                tot_bins_multiplier = 1
                udvs_denom = 1
Unknown's avatar
Unknown committed
101

Somnath, Suhas's avatar
Somnath, Suhas committed
102
103
104
        else:
            spec_label = 'None'
            parm_dict['data_type'] = 'BELineData'
Unknown's avatar
Unknown committed
105

Somnath, Suhas's avatar
Somnath, Suhas committed
106
107
        # Check file sizes:
        if 'read_real' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
108
109
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
Somnath, Suhas's avatar
Somnath, Suhas committed
110
111
112
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])
Unknown's avatar
Unknown committed
113

Somnath, Suhas's avatar
Somnath, Suhas committed
114
115
116
        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

Unknown's avatar
Unknown committed
117
        add_pix = False
Somnath, Suhas's avatar
Somnath, Suhas committed
118
119
        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
Unknown's avatar
Unknown committed
120
121
        num_pix = num_rows * num_cols
        tot_bins = real_size / (num_pix * 4)
Chris Smith's avatar
Chris Smith committed
122
        # Check for case where only a single pixel is missing.
Unknown's avatar
Unknown committed
123
124
125
        check_bins = real_size / ((num_pix - 1) * 4)

        if tot_bins % 1 and check_bins % 1:
126
            raise ValueError('Aborting! Some parameter appears to have changed in-between')
Somnath, Suhas's avatar
Somnath, Suhas committed
127
        elif not tot_bins % 1:
Chris Smith's avatar
Chris Smith committed
128
            # Everything's ok
Somnath, Suhas's avatar
Somnath, Suhas committed
129
130
131
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
Unknown's avatar
Unknown committed
132
133
134
135
136
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.')
            add_pix = True

        tot_bins = int(tot_bins) * tot_bins_multiplier

Somnath, Suhas's avatar
Somnath, Suhas committed
137
        if 'parm_mat' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
138
            (bin_inds, bin_freqs, bin_FFT, ex_wfm) = self.__read_parms_mat(path_dict['parm_mat'], isBEPS)
Somnath, Suhas's avatar
Somnath, Suhas committed
139
        elif 'old_mat_parms' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
140
            (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__read_old_mat_be_vecs(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
141
        else:
Unknown's avatar
Unknown committed
142
            band_width = parm_dict['BE_band_width_[Hz]'] * (0.5 - parm_dict['BE_band_edge_trim'])
Somnath, Suhas's avatar
Somnath, Suhas committed
143
            st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
Unknown's avatar
Unknown committed
144
            en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width
Somnath, Suhas's avatar
Somnath, Suhas committed
145
            bin_freqs = np.linspace(st_f, en_f, tot_bins, dtype=np.float32)
Unknown's avatar
Unknown committed
146

147
            warn('No parms .mat file found.... Filling dummy values into ancillary datasets.')
Somnath, Suhas's avatar
Somnath, Suhas committed
148
149
150
            bin_inds = np.zeros(shape=tot_bins, dtype=np.int32)
            bin_FFT = np.zeros(shape=tot_bins, dtype=np.complex64)
            ex_wfm = np.zeros(shape=100, dtype=np.float32)
Unknown's avatar
Unknown committed
151

Somnath, Suhas's avatar
Somnath, Suhas committed
152
153
154
155
156
        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)
157

Somnath, Suhas's avatar
Somnath, Suhas committed
158
        self.FFT_BE_wave = bin_FFT
159

Somnath, Suhas's avatar
Somnath, Suhas committed
160
        if isBEPS:
Somnath, Suhas's avatar
Somnath, Suhas committed
161
            (UDVS_labs, UDVS_units, UDVS_mat) = self.__build_udvs_table(parm_dict)
Unknown's avatar
Unknown committed
162
163

            #             Remove the unused plot group columns before proceeding:
Somnath, Suhas's avatar
Somnath, Suhas committed
164
            (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
Unknown's avatar
Unknown committed
165

166
            old_spec_inds = np.zeros(shape=(2, tot_bins), dtype=INDICES_DTYPE)
Unknown's avatar
Unknown committed
167
168
169
170
171

            #             Will assume that all excitation waveforms have same number of bins
            num_actual_udvs_steps = UDVS_mat.shape[0] / udvs_denom
            bins_per_step = tot_bins / num_actual_udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
172
            if bins_per_step % 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
173
174
                print('UDVS mat shape: {}, total bins: {}, bins per step: {}'.format(UDVS_mat.shape, tot_bins,
                                                                                     bins_per_step))
175
                raise ValueError('Non integer number of bins per step!')
Unknown's avatar
Unknown committed
176

Somnath, Suhas's avatar
Somnath, Suhas committed
177
178
            bins_per_step = int(bins_per_step)
            num_actual_udvs_steps = int(num_actual_udvs_steps)
Unknown's avatar
Unknown committed
179
180
181

            stind = 0
            for step_index in range(UDVS_mat.shape[0]):
Unknown's avatar
Unknown committed
182
183
184
                if UDVS_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                    continue
                # Bin step
185
                old_spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=INDICES_DTYPE)
Unknown's avatar
Unknown committed
186
                # UDVS step
187
                old_spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step, dtype=INDICES_DTYPE)
Somnath, Suhas's avatar
Somnath, Suhas committed
188
                stind += bins_per_step
Somnath, Suhas's avatar
Somnath, Suhas committed
189
            del stind, step_index
Unknown's avatar
Unknown committed
190

Somnath, Suhas's avatar
Somnath, Suhas committed
191
        else:  # BE Line
Somnath, Suhas's avatar
Somnath, Suhas committed
192
            self.signal_type = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
193
            self.expt_type = 1  # Stephen has not used this index for some reason
Somnath, Suhas's avatar
Somnath, Suhas committed
194
195
            num_actual_udvs_steps = 1
            bins_per_step = tot_bins
Somnath, Suhas's avatar
Somnath, Suhas committed
196
            UDVS_labs = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'be-line']
Somnath, Suhas's avatar
Somnath, Suhas committed
197
            UDVS_units = ['', 'V', 'A', '', '', '']
Somnath, Suhas's avatar
Somnath, Suhas committed
198
199
            UDVS_mat = np.array([1, 0, parm_dict['BE_amplitude_[V]'], 1, 1, 1],
                                dtype=np.float32).reshape(1, len(UDVS_labs))
Somnath, Suhas's avatar
Somnath, Suhas committed
200

201
            old_spec_inds = np.vstack((np.arange(tot_bins, dtype=INDICES_DTYPE), np.zeros(tot_bins, dtype=INDICES_DTYPE)))
Unknown's avatar
Unknown committed
202

Somnath, Suhas's avatar
Somnath, Suhas committed
203
204
205
        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
206
        parm_dict['num_udvs_steps'] = num_actual_udvs_steps
Unknown's avatar
Unknown committed
207

Somnath, Suhas's avatar
Somnath, Suhas committed
208
        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
209
        for col_ind, col_name in enumerate(UDVS_labs):
Unknown's avatar
Unknown committed
210
211
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind + 1))

Somnath, Suhas's avatar
Somnath, Suhas committed
212
        # Need to add the Bin Waveform type - infer from UDVS        
Unknown's avatar
Unknown committed
213
        exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
Somnath, Suhas's avatar
Somnath, Suhas committed
214
215
216

        if self.expt_type == 2:
            # Need to double the vectors:
Unknown's avatar
Unknown committed
217
            exec_bin_vec = np.hstack((exec_bin_vec, -1 * exec_bin_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
218
219
            bin_inds = np.hstack((bin_inds, bin_inds))
            bin_freqs = np.hstack((bin_freqs, bin_freqs))
Somnath, Suhas's avatar
Somnath, Suhas committed
220
            # This is wrong but I don't know what else to do
Somnath, Suhas's avatar
Somnath, Suhas committed
221
            bin_FFT = np.hstack((bin_FFT, bin_FFT))
Unknown's avatar
Unknown committed
222

Somnath, Suhas's avatar
Somnath, Suhas committed
223
        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
224
        # This is an old and legacy way of doing things. Ideally, all we would need ot do is just get the unit values
Somnath, Suhas's avatar
Somnath, Suhas committed
225
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_labs_names = createSpecVals(UDVS_mat,
226
                                                                                                     old_spec_inds,
Somnath, Suhas's avatar
Somnath, Suhas committed
227
228
229
230
231
                                                                                                     bin_freqs,
                                                                                                     exec_bin_vec,
                                                                                                     parm_dict,
                                                                                                     UDVS_labs,
                                                                                                     UDVS_units)
232
233
234
235
        # Not sure what is happening here but this should work.
        spec_dim_dict = dict()
        for entry in spec_vals_labs_names:
            spec_dim_dict[entry[0] + '_parameters'] = entry[1]
Chris Smith's avatar
Chris Smith committed
236

Somnath, Suhas's avatar
Somnath, Suhas committed
237
        spec_vals_slices = dict()
Unknown's avatar
Unknown committed
238
239
240
        #         if len(spec_vals_labs) == 1:
        #             spec_vals_slices[spec_vals_labs[0]]=(slice(0,1,None),)
        #         else:
Somnath, Suhas's avatar
Somnath, Suhas committed
241
242

        for row_ind, row_name in enumerate(spec_vals_labs):
Unknown's avatar
Unknown committed
243
            spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
Somnath, Suhas's avatar
Somnath, Suhas committed
244

245
246
        if path.exists(h5_path):
            remove(h5_path)
Chris Smith's avatar
Chris Smith committed
247

248
249
        # First create the file
        h5_f = h5py.File(h5_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
250

251
        # Then write root level attributes
252
        global_parms = generate_dummy_main_parms()
Somnath, Suhas's avatar
Somnath, Suhas committed
253
254
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
255
256
257
258
        try:
            global_parms['experiment_date'] = parm_dict['File_date_and_time']
        except KeyError:
            global_parms['experiment_date'] = '1:1:1'
Chris Smith's avatar
Chris Smith committed
259

Somnath, Suhas's avatar
Somnath, Suhas committed
260
        # assuming that the experiment was completed:
Unknown's avatar
Unknown committed
261
262
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
Somnath, Suhas's avatar
Somnath, Suhas committed
263
        global_parms['data_type'] = parm_dict['data_type']
Somnath, Suhas's avatar
Somnath, Suhas committed
264
        global_parms['translator'] = 'ODF'
265
        write_simple_attrs(h5_f, global_parms)
266
        write_book_keeping_attrs(h5_f)
Unknown's avatar
Unknown committed
267

268
269
        # Then create the measurement group
        h5_meas_group = create_indexed_group(h5_f, 'Measurement')
Unknown's avatar
Unknown committed
270

271
272
        # Write attributes at the measurement group level
        write_simple_attrs(h5_meas_group, parm_dict)
Unknown's avatar
Unknown committed
273

274
275
        # Create the Channel group
        h5_chan_grp = create_indexed_group(h5_meas_group, 'Channel')
Unknown's avatar
Unknown committed
276

277
278
        # Write channel group attributes
        write_simple_attrs(h5_chan_grp, {'Channel_Input': 'IO_Analog_Input_1'})
Unknown's avatar
Unknown committed
279

280
281
        # Now the datasets!
        h5_ex_wfm = h5_chan_grp.create_dataset('Excitation_Waveform', data=ex_wfm)
Unknown's avatar
Unknown committed
282

283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
        h5_udvs = h5_chan_grp.create_dataset('UDVS', data=UDVS_mat)
        write_region_references(h5_udvs, udvs_slices, add_labels_attr=True, verbose=verbose)
        write_simple_attrs(h5_udvs, {'units': UDVS_units}, verbose=verbose)
        
        # ds_udvs_labs = MicroDataset('UDVS_Labels',np.array(UDVS_labs))
        h5_UDVS_inds = h5_chan_grp.create_dataset('UDVS_Indices', data=old_spec_inds[1])

        # ds_spec_labs = MicroDataset('Spectroscopic_Labels',np.array(['Bin','UDVS_Step']))
        h5_bin_steps = h5_chan_grp.create_dataset('Bin_Step', data=np.arange(bins_per_step, dtype=INDICES_DTYPE), 
                                                  dtype=INDICES_DTYPE)

        h5_bin_inds = h5_chan_grp.create_dataset('Bin_Indices', data=bin_inds, dtype=INDICES_DTYPE)
        h5_bin_freq = h5_chan_grp.create_dataset('Bin_Frequencies', data=bin_freqs)
        h5_bin_FFT = h5_chan_grp.create_dataset('Bin_FFT', data=bin_FFT)
        h5_wfm_typ = h5_chan_grp.create_dataset('Bin_Wfm_Type', data=exec_bin_vec)

        pos_dims = [Dimension('X', 'm', np.arange(num_cols)), Dimension('Y', 'm', np.arange(num_rows))]
        h5_pos_ind, h5_pos_val = write_ind_val_dsets(h5_chan_grp, pos_dims, is_spectral=False, verbose=verbose)

        h5_spec_inds = h5_chan_grp.create_dataset('Spectroscopic_Indices', data=spec_inds, dtype=INDICES_DTYPE)        
        h5_spec_vals = h5_chan_grp.create_dataset('Spectroscopic_Values', data=np.array(spec_vals), dtype=VALUES_DTYPE)
        for dset in [h5_spec_inds, h5_spec_vals]:
            write_region_references(dset, spec_vals_slices, add_labels_attr=True, verbose=verbose)
            write_simple_attrs(dset, {'units': spec_vals_units}, verbose=verbose)
307
            write_simple_attrs(dset, spec_dim_dict)
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326

        # Noise floor should be of shape: (udvs_steps x 3 x positions)
        h5_noise_floor = h5_chan_grp.create_dataset('Noise_Floor', (num_pix, num_actual_udvs_steps), dtype=nf32,
                                                    chunks=(1, num_actual_udvs_steps))

        """
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares
        of UDVS steps by pixels.  N is determined dynamically based on the dimensions of the
        dataset.  Currently it is set such that individual chunks are less than 10kB in size.

        Chris Smith -- csmith55@utk.edu
        """
        BEPS_chunks = calc_chunks([num_pix, tot_bins],
                                  np.complex64(0).itemsize,
                                  unit_chunks=(1, bins_per_step))
        self.h5_raw = write_main_dataset(h5_chan_grp, (num_pix, tot_bins), 'Raw_Data', 'Piezoresponse', 'V', None, None,
                                         dtype=np.complex64, chunks=BEPS_chunks, compression='gzip',
                                         h5_pos_inds=h5_pos_ind, h5_pos_vals=h5_pos_val, h5_spec_inds=h5_spec_inds,
                                         h5_spec_vals=h5_spec_vals, verbose=verbose)
Somnath, Suhas's avatar
Somnath, Suhas committed
327

Chris Smith's avatar
Chris Smith committed
328
        self._read_data(UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix)
Unknown's avatar
Unknown committed
329

330
        generatePlotGroups(self.h5_raw, self.mean_resp, folder_path, basename,
Somnath, Suhas's avatar
Somnath, Suhas committed
331
                           self.max_resp, self.min_resp, max_mem_mb=self.max_ram,
Somnath, Suhas's avatar
Somnath, Suhas committed
332
                           spec_label=spec_label, show_plots=show_plots, save_plots=save_plots,
Unknown's avatar
Unknown committed
333
                           do_histogram=do_histogram, debug=verbose)
Unknown's avatar
Unknown committed
334

335
        h5_f.close()
Unknown's avatar
Unknown committed
336

Somnath, Suhas's avatar
Somnath, Suhas committed
337
        return h5_path
Chris Smith's avatar
Chris Smith committed
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365

    def _read_data(self, UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix):
        """
        Checks if the data is BEPS or BELine and calls the correct function to read the data from
        file

        Parameters
        ----------
        UDVS_mat : numpy.ndarray of float
            UDVS table
        parm_dict : dict
            Experimental parameters
        path_dict : dict
            Dictionary of data files to be read
        real_size : dict
            Size of each data file
        isBEPS : boolean
            Is the data BEPS
        add_pix : boolean
            Does the reader need to add extra pixels to the end of the dataset

        Returns
        -------
        None
        """
        # Now read the raw data files:
        if not isBEPS:
            # Do this for all BE-Line (always small enough to read in one shot)
366
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
367
368
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'out-of-field':
            # Do this for out-of-field BEPS ONLY that is also small (256 MB)
369
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
370
371
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'in-field':
            # Do this for in-field only
372
            self.__quick_read_data(path_dict['write_real'], path_dict['write_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
373
374
        else:
            # Large BEPS datasets OR those with in-and-out of field
Somnath, Suhas's avatar
Somnath, Suhas committed
375
            self.__read_beps_data(path_dict, UDVS_mat.shape[0], parm_dict['VS_measure_in_field_loops'], add_pix)
376
        self.h5_raw.file.flush()
Chris Smith's avatar
Chris Smith committed
377

Somnath, Suhas's avatar
Somnath, Suhas committed
378
    def __read_beps_data(self, path_dict, udvs_steps, mode, add_pixel=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
        """
        Reads the imaginary and real data files pixelwise and writes to the H5 file 
        
        Parameters 
        --------------------
        path_dict : dictionary
            Dictionary containing the absolute paths of the real and imaginary data files
        udvs_steps : unsigned int
            Number of UDVS steps
        mode : String / Unicode
            'in-field', 'out-of-field', or 'in and out-of-field'
        add_pixel : boolean. (Optional; default is False)
            If an empty pixel worth of data should be written to the end             
        
        Returns 
        -------------------- 
        None
        """
Unknown's avatar
Unknown committed
397

Somnath, Suhas's avatar
Somnath, Suhas committed
398
        print('---- reading pixel-by-pixel ----------')
Unknown's avatar
Unknown committed
399
400
401
402

        bytes_per_pix = self.h5_raw.shape[1] * 4
        step_size = self.h5_raw.shape[1] / udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
403
        if mode == 'out-of-field':
Unknown's avatar
Unknown committed
404
            parsers = [BEodfParser(path_dict['read_real'], path_dict['read_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
405
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
406
        elif mode == 'in-field':
Unknown's avatar
Unknown committed
407
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
408
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
409
410
        elif mode == 'in and out-of-field':
            # each file will only have half the udvs steps:
Unknown's avatar
Unknown committed
411
            if 0.5 * udvs_steps % 1:
412
413
                raise ValueError('Odd number of UDVS')

Unknown's avatar
Unknown committed
414
            udvs_steps = int(0.5 * udvs_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
415
            # be careful - each pair contains only half the necessary bins - so read half
Unknown's avatar
Unknown committed
416
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
417
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2)),
Unknown's avatar
Unknown committed
418
419
420
                       BEodfParser(path_dict['read_real'], path_dict['read_imag'],
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2))]

Somnath, Suhas's avatar
Somnath, Suhas committed
421
            if step_size % 1:
422
423
                raise ValueError('strange number of bins per UDVS step. Exiting')

Somnath, Suhas's avatar
Somnath, Suhas committed
424
            step_size = int(step_size)
425

426
427
        rand_spectra = self.__get_random_spectra(parsers, self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
428
        take_conjugate = requires_conjugate(rand_spectra)
429

Somnath, Suhas's avatar
Somnath, Suhas committed
430
431
432
433
        self.mean_resp = np.zeros(shape=(self.h5_raw.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)

Unknown's avatar
Unknown committed
434
        numpix = self.h5_raw.shape[0]
Somnath, Suhas's avatar
Somnath, Suhas committed
435
436
437
        """ 
        Don't try to do the last step if a pixel is missing.   
        This will be handled after the loop. 
Unknown's avatar
Unknown committed
438
439
440
441
        """
        if add_pixel:
            numpix -= 1

Somnath, Suhas's avatar
Somnath, Suhas committed
442
        for pix_indx in range(numpix):
Somnath, Suhas's avatar
Somnath, Suhas committed
443
            if self.h5_raw.shape[0] > 5:
Unknown's avatar
Unknown committed
444
445
446
                if pix_indx % int(round(self.h5_raw.shape[0] / 10)) == 0:
                    print('Reading... {} complete'.format(round(100 * pix_indx / self.h5_raw.shape[0])))

Somnath, Suhas's avatar
Somnath, Suhas committed
447
448
449
            # get the raw stream from each parser
            pxl_data = list()
            for prsr in parsers:
Somnath, Suhas's avatar
Somnath, Suhas committed
450
                pxl_data.append(prsr.read_pixel())
Unknown's avatar
Unknown committed
451

Somnath, Suhas's avatar
Somnath, Suhas committed
452
453
454
455
456
            # interleave if both in and out of field
            # we are ignoring user defined possibilities...
            if mode == 'in and out-of-field':
                in_fld = pxl_data[0]
                out_fld = pxl_data[1]
Unknown's avatar
Unknown committed
457

Somnath, Suhas's avatar
Somnath, Suhas committed
458
459
                in_fld_2 = in_fld.reshape(udvs_steps, step_size)
                out_fld_2 = out_fld.reshape(udvs_steps, step_size)
Unknown's avatar
Unknown committed
460
                raw_mat = np.empty((udvs_steps * 2, step_size), dtype=out_fld.dtype)
Somnath, Suhas's avatar
Somnath, Suhas committed
461
462
                raw_mat[0::2, :] = in_fld_2
                raw_mat[1::2, :] = out_fld_2
Somnath, Suhas's avatar
Somnath, Suhas committed
463
464
                raw_vec = raw_mat.reshape(in_fld.size + out_fld.size).transpose()
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
465
                raw_vec = pxl_data[0]  # only one parser
Somnath, Suhas's avatar
Somnath, Suhas committed
466
467
            self.max_resp[pix_indx] = np.max(np.abs(raw_vec))
            self.min_resp[pix_indx] = np.min(np.abs(raw_vec))
Unknown's avatar
Unknown committed
468
            self.mean_resp = (1 / (pix_indx + 1)) * (raw_vec + pix_indx * self.mean_resp)
469
470
471

            if take_conjugate:
                raw_vec = np.conjugate(raw_vec)
472
            self.h5_raw[pix_indx, :] = np.complex64(raw_vec[:])
473
            self.h5_raw.file.flush()
Unknown's avatar
Unknown committed
474

Somnath, Suhas's avatar
Somnath, Suhas committed
475
        # Add zeros to main_data for the missing pixel. 
Unknown's avatar
Unknown committed
476
477
478
        if add_pixel:
            self.h5_raw[-1, :] = 0 + 0j

Somnath, Suhas's avatar
Somnath, Suhas committed
479
        print('---- Finished reading files -----')
480
481

    def __quick_read_data(self, real_path, imag_path, udvs_steps):
Somnath, Suhas's avatar
Somnath, Suhas committed
482
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
483
484
485
486
487
488
489
490
        Returns information about the excitation BE waveform present in the .mat file

        Parameters
        -----------
        real_path : String / Unicode
            Absolute file path of the real data file
        imag_path : String / Unicode
            Absolute file path of the real data file
491
492
        udvs_steps : unsigned int
            Number of UDVS steps
Somnath, Suhas's avatar
Somnath, Suhas committed
493
        """
Unknown's avatar
Unknown committed
494
        print('---- reading all data at once ----------')
Somnath, Suhas's avatar
Somnath, Suhas committed
495

Unknown's avatar
Unknown committed
496
        parser = BEodfParser(real_path, imag_path, self.h5_raw.shape[0], self.h5_raw.shape[1] * 4)
497
498

        step_size = self.h5_raw.shape[1] / udvs_steps
499
500
        rand_spectra = self.__get_random_spectra([parser], self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
501
        take_conjugate = requires_conjugate(rand_spectra)
Somnath, Suhas's avatar
Somnath, Suhas committed
502
        raw_vec = parser.read_all_data()
503
        if take_conjugate:
504
            print('Taking conjugate to ensure positive Quality factors')
505
            raw_vec = np.conjugate(raw_vec)
Unknown's avatar
Unknown committed
506

Somnath, Suhas's avatar
Somnath, Suhas committed
507
        raw_mat = raw_vec.reshape(self.h5_raw.shape[0], self.h5_raw.shape[1])
Unknown's avatar
Unknown committed
508

Somnath, Suhas's avatar
Somnath, Suhas committed
509
        # Write to the h5 dataset:
Somnath, Suhas's avatar
Somnath, Suhas committed
510
511
512
        self.mean_resp = np.mean(raw_mat, axis=0)
        self.max_resp = np.amax(np.abs(raw_mat), axis=0)
        self.min_resp = np.amin(np.abs(raw_mat), axis=0)
513
        self.h5_raw[:, :] = np.complex64(raw_mat)
514
        self.h5_raw.file.flush()
Somnath, Suhas's avatar
Somnath, Suhas committed
515

Unknown's avatar
Unknown committed
516
517
        print('---- Finished reading files -----')

518
    def _parse_file_path(self, data_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
519
520
521
522
523
524
525
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary
        
        Parameters 
        --------------------
        data_filepath: String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
526
            Absolute path of any file in the same directory as the .dat files
Somnath, Suhas's avatar
Somnath, Suhas committed
527
528
529
530
531
532
533
534
535
        
        Returns 
        --------------------
        basename : String / Unicode
            Basename of the dataset      
        path_dict : Dictionary
            Dictionary containing absolute paths of all necessary data and parameter files
        """
        (folder_path, basename) = path.split(data_filepath)
Unknown's avatar
Unknown committed
536
        (super_folder, basename) = path.split(folder_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
537
538
539
540
541
542
543
544
545
546
547

        if basename.endswith('_d'):
            # Old old data format where the folder ended with a _d for some reason
            basename = basename[:-2]
        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()
Unknown's avatar
Unknown committed
548

Somnath, Suhas's avatar
Somnath, Suhas committed
549
        for file_name in listdir(folder_path):
Chris Smith's avatar
Chris Smith committed
550
            abs_path = path.join(folder_path, file_name)
Somnath, Suhas's avatar
Somnath, Suhas committed
551
552
553
554
555
            if file_name.endswith('.txt') and file_name.find('parm') > 0:
                path_dict['parm_txt'] = abs_path
            elif file_name.find('.mat') > 0:
                if file_name.find('more_parms') > 0:
                    path_dict['parm_mat'] = abs_path
Unknown's avatar
Unknown committed
556
                elif file_name == (basename + '.mat'):
Somnath, Suhas's avatar
Somnath, Suhas committed
557
558
559
560
561
562
563
564
565
566
567
568
                    path_dict['old_mat_parms'] = abs_path
            elif file_name.endswith('.dat'):
                # Need to account for the second AI channel here
                file_tag = 'read'
                if file_name.find('write') > 0:
                    file_tag = 'write'
                if file_name.find('real') > 0:
                    file_tag += '_real'
                elif file_name.find('imag') > 0:
                    file_tag += '_imag'
                path_dict[file_tag] = abs_path

Chris Smith's avatar
Chris Smith committed
569
        return basename, path_dict
Somnath, Suhas's avatar
Somnath, Suhas committed
570
571
572

    @staticmethod
    def __read_old_mat_be_vecs(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
        """
        Returns information about the excitation BE waveform present in the 
        more parms.mat file
        
        Parameters 
        --------------------
        filepath : String or unicode
            Absolute filepath of the .mat parameter file
        
        Returns 
        --------------------
        bin_inds : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        bin_w : 1D numpy float array
            Excitation bin Frequencies
        bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        BE_wave : 1D numpy float array
            Band Excitation waveform
        dc_amp_vec_full : 1D numpy float array
            spectroscopic waveform. 
            This information will be necessary for fixing the UDVS for AC modulation for example
        """
Unknown's avatar
Unknown committed
596
        matread = loadmat(file_path, squeeze_me=True)
Somnath, Suhas's avatar
Somnath, Suhas committed
597
        BE_wave = matread['BE_wave']
Unknown's avatar
Unknown committed
598
        bin_inds = matread['bin_ind'] - 1  # Python base 0
Somnath, Suhas's avatar
Somnath, Suhas committed
599
600
601
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
Somnath, Suhas's avatar
Somnath, Suhas committed
602
603
        bin_FFT = np.conjugate(FFT_full[bin_inds])
        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full
Unknown's avatar
Unknown committed
604

Somnath, Suhas's avatar
Somnath, Suhas committed
605
606
    @staticmethod
    def __get_parms_from_old_mat(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it
        
        Parameters 
        --------------------
        file_path : Unicode / String
            absolute filepath of the .mat file containing the parameters
            
        Returns 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)
Unknown's avatar
Unknown committed
623
624
625

        parm_dict['IO_rate'] = str(int(matread['AO_rate'] / 1E+6)) + ' MHz'

Somnath, Suhas's avatar
Somnath, Suhas committed
626
627
628
629
630
        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = position_vec[2]
        parm_dict['grid_num_cols'] = position_vec[3]
Unknown's avatar
Unknown committed
631

Somnath, Suhas's avatar
Somnath, Suhas committed
632
633
        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
Somnath, Suhas's avatar
Somnath, Suhas committed
634
            parm_dict['grid_num_rows'] = position_vec[0]  # set to number of present cols and rows
Somnath, Suhas's avatar
Somnath, Suhas committed
635
            parm_dict['grid_num_cols'] = position_vec[1]
Unknown's avatar
Unknown committed
636

Somnath, Suhas's avatar
Somnath, Suhas committed
637
638
639
640
641
642
643
644
645
        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
646
647
        parm_dict['BE_band_edge_smoothing_[s]'] = BE_parm_vec_1[4]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
Unknown's avatar
Unknown committed
648
649
650
        parm_dict['BE_window_adjustment'] = BE_parm_vec_1[6]
        parm_dict['BE_points_per_step'] = 2 ** int(BE_parm_vec_1[7])
        parm_dict['BE_repeats'] = 2 ** int(BE_parm_vec_1[8])
Somnath, Suhas's avatar
Somnath, Suhas committed
651
652
653
654
        try:
            parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        except KeyError:
            parm_dict['BE_bins_per_read'] = len(matread['bin_w'])
Unknown's avatar
Unknown committed
655

Somnath, Suhas's avatar
Somnath, Suhas committed
656
        assembly_parm_vec = matread['assembly_parm_vec']
Unknown's avatar
Unknown committed
657

Somnath, Suhas's avatar
Somnath, Suhas committed
658
659
660
661
662
663
        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
Unknown's avatar
Unknown committed
664

Somnath, Suhas's avatar
Somnath, Suhas committed
665
666
667
668
669
        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'
Unknown's avatar
Unknown committed
670

Somnath, Suhas's avatar
Somnath, Suhas committed
671
672
        # num_driving_bands = assembly_parm_vec[0]  # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1]  # 0 parallel 1 series
Unknown's avatar
Unknown committed
673

Somnath, Suhas's avatar
Somnath, Suhas committed
674
675
        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']
Unknown's avatar
Unknown committed
676
677
678
679

        VS_start_V = VS_parms[4]
        VS_start_loop_amp = VS_parms[5]
        VS_final_loop_amp = VS_parms[6]
Somnath, Suhas's avatar
Somnath, Suhas committed
680
        # VS_read_write_ratio = VS_parms[8]  # 1 <- SS_read_write_ratio
Unknown's avatar
Unknown committed
681

Somnath, Suhas's avatar
Somnath, Suhas committed
682
        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
Unknown's avatar
Unknown committed
683
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
684
685
        parm_dict['VS_steps_per_full_cycle'] = VS_parms[7]
        parm_dict['VS_cycle_fraction'] = 'full'
Unknown's avatar
Unknown committed
686
        parm_dict['VS_cycle_phase_shift'] = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
687
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
Somnath, Suhas's avatar
Somnath, Suhas committed
688
689
690
691
692
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0
Unknown's avatar
Unknown committed
693

Somnath, Suhas's avatar
Somnath, Suhas committed
694
695
        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
Unknown's avatar
Unknown committed
696
697
698
            parm_dict['VS_amplitude_[V]'] = 0.5 * (
                max(dc_amp_vec_full) - min(dc_amp_vec_full))  # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)
Somnath, Suhas's avatar
Somnath, Suhas committed
699
700
701
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
Somnath, Suhas's avatar
Somnath, Suhas committed
702
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
703
            parm_dict['VS_offset_[V]'] = 0
Unknown's avatar
Unknown committed
704
            parm_dict['VS_number_of_cycles'] = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
705
706
707
708
709
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
710
711
712
        elif VS_parms[0] == 2:
            # AC mode 
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
Somnath, Suhas's avatar
Somnath, Suhas committed
713
714
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
            parm_dict['VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
Somnath, Suhas's avatar
Somnath, Suhas committed
715
716
        else:
            parm_dict['VS_mode'] = 'Custom'
Unknown's avatar
Unknown committed
717

Somnath, Suhas's avatar
Somnath, Suhas committed
718
        return parm_dict
Unknown's avatar
Unknown committed
719

Somnath, Suhas's avatar
Somnath, Suhas committed
720
721
    @staticmethod
    def __read_parms_mat(file_path, is_beps):
Somnath, Suhas's avatar
Somnath, Suhas committed
722
723
724
725
726
        """
        Returns information about the excitation BE waveform present in the more parms.mat file
        
        Parameters 
        --------------------
Somnath, Suhas's avatar
Somnath, Suhas committed
727
        file_path : String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
728
            Absolute filepath of the .mat parameter file
Somnath, Suhas's avatar
Somnath, Suhas committed
729
        is_beps : Boolean
Somnath, Suhas's avatar
Somnath, Suhas committed
730
731
732
733
734
735
736
737
738
739
740
741
742
            Whether or not this is BEPS or BE-Line
        
        Returns 
        --------------------
        BE_bin_ind : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        BE_bin_w : 1D numpy float array
            Excitation bin Frequencies
        BE_bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        ex_wfm : 1D numpy float array
            Band Excitation waveform
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
743
        if not path.exists(file_path):
744
            raise IOError('NO "More parms" file found')
Somnath, Suhas's avatar
Somnath, Suhas committed
745
        if is_beps:
Somnath, Suhas's avatar
Somnath, Suhas committed
746
747
748
            fft_name = 'FFT_BE_wave'
        else:
            fft_name = 'FFT_BE_rev_wave'
Somnath, Suhas's avatar
Somnath, Suhas committed
749
        matread = loadmat(file_path, variable_names=['BE_bin_ind', 'BE_bin_w', fft_name])
Unknown's avatar
Unknown committed
750
        BE_bin_ind = np.squeeze(matread['BE_bin_ind']) - 1  # From Matlab (base 1) to Python (base 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
751
752
        BE_bin_w = np.squeeze(matread['BE_bin_w'])
        FFT_full = np.complex64(np.squeeze(matread[fft_name]))
Somnath, Suhas's avatar
Somnath, Suhas committed
753
        # For whatever weird reason, the sign of the imaginary portion is flipped. Correct it:
Unknown's avatar
Unknown committed
754
        # BE_bin_FFT = np.conjugate(FFT_full[BE_bin_ind])
Somnath, Suhas's avatar
Somnath, Suhas committed
755
756
        BE_bin_FFT = np.zeros(len(BE_bin_ind), dtype=np.complex64)
        BE_bin_FFT.real = np.real(FFT_full[BE_bin_ind])
Unknown's avatar
Unknown committed
757
758
        BE_bin_FFT.imag = -1 * np.imag(FFT_full[BE_bin_ind])

Somnath, Suhas's avatar
Somnath, Suhas committed
759
        ex_wfm = np.real(np.fft.ifft(np.fft.ifftshift(FFT_full)))
Somnath, Suhas's avatar
Somnath, Suhas committed
760
761

        return BE_bin_ind, BE_bin_w, BE_bin_FFT, ex_wfm
Unknown's avatar
Unknown committed
762

Somnath, Suhas's avatar
Somnath, Suhas committed
763
    def __build_udvs_table(self, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
        """
        Generates the UDVS table using the parameters
        
        Parameters 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        
        Returns 
        --------------------      
        UD_VS_table_label : List of strings
            Labels for columns in the UDVS table
        UD_VS_table_unit : List of strings
            Units for the columns in the UDVS table
        UD_VS_table : 2D numpy float array
            UDVS data table
        """
Unknown's avatar
Unknown committed
781

Somnath, Suhas's avatar
Somnath, Suhas committed
782
        def translate_val(target, strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
783
784
            """
            Internal function - Interprets the provided value using the provided lookup table
Somnath, Suhas's avatar
Somnath, Suhas committed
785
786
787
788
789
790
791
792
793

            Parameters
            ----------
            target : String
                Item we are looking for in the strvals list
            strvals : list of strings
                List of source values
            numvals : list of numbers
                List of results
Somnath, Suhas's avatar
Somnath, Suhas committed
794
            """
Unknown's avatar
Unknown committed
795

Somnath, Suhas's avatar
Somnath, Suhas committed
796
            if len(strvals) is not len(numvals):
Unknown's avatar
Unknown committed
797
                return None
Somnath, Suhas's avatar
Somnath, Suhas committed
798
            for strval, fltval in zip(strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
799
800
                if target == strval:
                    return fltval
Somnath, Suhas's avatar
Somnath, Suhas committed
801
            return None  # not found in list
Unknown's avatar
Unknown committed
802
803

        # % Extract values from parm text file
Unknown's avatar
Unknown committed
804
        BE_signal_type = translate_val(parm_dict['BE_phase_content'],
Unknown's avatar
Unknown committed
805
806
807
                                       ['chirp-sinc hybrid', '1/2 harmonic excitation',
                                        '1/3 harmonic excitation', 'pure sine'],
                                       [1, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
808
809
810
811
812
813
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type
        if BE_signal_type is 4:
            self.harmonic = 1
        BE_amp = parm_dict['BE_amplitude_[V]']
Unknown's avatar
Unknown committed
814

Somnath, Suhas's avatar
Somnath, Suhas committed
815
816
        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
Unknown's avatar
Unknown committed
817
        # VS_read_voltage = parm_dict['VS_read_voltage_[V]']
Somnath, Suhas's avatar
Somnath, Suhas committed
818
819
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
Somnath, Suhas's avatar
Somnath, Suhas committed
820
821
822
        VS_fraction = translate_val(parm_dict['VS_cycle_fraction'],
                                    ['full', '1/2', '1/4', '3/4'],
                                    [1., 0.5, 0.25, 0.75])
Somnath, Suhas's avatar
Somnath, Suhas committed
823
824
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
825
826
827
828
829
830
831
            VS_shift = translate_val(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
        VS_in_out_cond = translate_val(parm_dict['VS_measure_in_field_loops'],
                                       ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
        VS_ACDC_cond = translate_val(parm_dict['VS_mode'],
                                     ['DC modulation mode', 'AC modulation mode with time reversal',
                                      'load user defined VS Wave from file', 'current mode'],
                                     [0, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
832
833
834
835
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
Unknown's avatar
Unknown committed
836
        # FORC_repeats = parm_dict['# of FORC repeats']
Somnath, Suhas's avatar
Somnath, Suhas committed
837
838
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']
Unknown's avatar
Unknown committed
839
840
841

        # % build vector of voltage spectroscopy values

Somnath, Suhas's avatar
Somnath, Suhas committed
842
        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
Unknown's avatar
Unknown committed
843
            VS_amp_vec_1 = np.arange(0, 1 + 1 / (VS_steps / 4), 1 / (VS_steps / 4))
Somnath, Suhas's avatar
Somnath, Suhas committed
844
845
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
Unknown's avatar
Unknown committed
846
847
            VS_amp_vec_4 = VS_amp_vec_1[1:-1] - 1
            vs_amp_vec = VS_amp * (np.hstack((VS_amp_vec_1, VS_amp_vec_2, VS_amp_vec_3, VS_amp_vec_4)))
Unknown's avatar
Unknown committed
848
            # apply phase shift to VS wave
Unknown's avatar
Unknown committed
849
            vs_amp_vec = np.roll(vs_amp_vec, int(np.floor(VS_steps / VS_fraction * VS_shift)))
Unknown's avatar
Unknown committed
850
            # cut VS waveform
Unknown's avatar
Unknown committed
851
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction))]
Unknown's avatar
Unknown committed
852
            # repeat VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
853
            vs_amp_vec = np.tile(vs_amp_vec, int(VS_cycles))
Unknown's avatar
Unknown committed
854
855
            vs_amp_vec = vs_amp_vec + VS_offset

Somnath, Suhas's avatar
Somnath, Suhas committed
856
        elif VS_ACDC_cond == 2:  # AC voltage spectroscopy with time reversal
Unknown's avatar
Unknown committed
857
858
            vs_amp_vec = VS_amp * np.arange(1 / (VS_steps / 2 / VS_fraction), 1 + 1 / (VS_steps / 2 / VS_fraction),
                                            1 / (VS_steps / 2 / VS_fraction))
Somnath, Suhas's avatar
Somnath, Suhas committed
859
            vs_amp_vec = np.roll(vs_amp_vec,
Unknown's avatar
Unknown committed
860
861
                                 int(np.floor(VS_steps / VS_fraction * VS_shift)))  # apply phase shift to VS wave
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction / 2))]  # cut VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
862
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles * 2)  # repeat VS waveform
Unknown's avatar
Unknown committed
863

Somnath, Suhas's avatar
Somnath, Suhas committed
864
        if FORC_cycles > 1:
Unknown's avatar
Unknown committed
865
866
867
868
869
870
871
            vs_amp_vec = vs_amp_vec / np.max(np.abs(vs_amp_vec))
            FORC_cycle_vec = np.arange(0, FORC_cycles + 1, FORC_cycles / (FORC_cycles - 1))
            FORC_A_vec = FORC_cycle_vec * (FORC_A2 - FORC_A1) / FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec * (FORC_B2 - FORC_B1) / FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec - FORC_B_vec) / 2
            FORC_off_vec = (FORC_A_vec + FORC_B_vec) / 2

Somnath, Suhas's avatar
Somnath, Suhas committed
872
873
874
            VS_amp_mat = np.tile(vs_amp_vec, [FORC_cycles, 1])
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(vs_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(vs_amp_vec), 1]).transpose()
Unknown's avatar
Unknown committed
875
876
877
            VS_amp_mat = VS_amp_mat * FORC_amp_mat + FORC_off_mat
            vs_amp_vec = VS_amp_mat.reshape(int(FORC_cycles * VS_cycles * VS_fraction * VS_steps))

Somnath, Suhas's avatar
Somnath, Suhas committed
878
879
        # Build UDVS table:
        if VS_ACDC_cond is 0 or VS_ACDC_cond is 4:  # DC voltage spectroscopy or current mode
Unknown's avatar
Unknown committed
880

Somnath, Suhas's avatar
Somnath, Suhas committed
881
            if VS_ACDC_cond is 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
882
                UD_dc_vec = np.vstack((vs_amp_vec, np.zeros(len(vs_amp_vec))))
Somnath, Suhas's avatar
Somnath, Suhas committed
883
            if VS_ACDC_cond is 4:
Somnath, Suhas's avatar
Somnath, Suhas committed
884
                UD_dc_vec = np.vstack((vs_amp_vec, vs_amp_vec))
Unknown's avatar
Unknown committed
885

Somnath, Suhas's avatar
Somnath, Suhas committed
886
            UD_dc_vec = UD_dc_vec.transpose().reshape(UD_dc_vec.size)
Somnath, Suhas's avatar
Somnath, Suhas committed
887
            num_VS_steps = UD_dc_vec.size
Unknown's avatar
Unknown committed
888

Somnath, Suhas's avatar
Somnath, Suhas committed
889
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'in-field', 'out-of-field']
Somnath, Suhas's avatar
Somnath, Suhas committed
890
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'V', 'V']
Somnath, Suhas's avatar
Somnath, Suhas committed
891
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Unknown's avatar
Unknown committed
892

Somnath, Suhas's avatar
Somnath, Suhas committed
893
894
            udvs_table[:, 0] = np.arange(0, num_VS_steps)  # Python base 0
            udvs_table[:, 1] = UD_dc_vec
Unknown's avatar
Unknown committed
895
896
897
898

            BE_IF_switch = np.abs(np.imag(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))
            BE_OF_switch = np.abs(np.real(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))

Somnath, Suhas's avatar
Somnath, Suhas committed
899
900
901
902
903
904
            if VS_in_out_cond is 0:  # out of field only
                udvs_table[:, 2] = BE_amp * BE_OF_switch
            elif VS_in_out_cond is 1:  # in field only
                udvs_table[:, 2] = BE_amp * BE_IF_switch
            elif VS_in_out_cond is 2:  # both in and out of field
                udvs_table[:, 2] = BE_amp * np.ones(num_VS_steps)
Unknown's avatar
Unknown committed
905

Somnath, Suhas's avatar
Somnath, Suhas committed
906
907
            udvs_table[:, 3] = np.ones(num_VS_steps)  # wave type
            udvs_table[:, 4] = np.ones(num_VS_steps) * BE_signal_type  # wave mod
Unknown's avatar
Unknown committed
908
909
910
911

            udvs_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN') * np.ones(num_VS_steps)

Somnath, Suhas's avatar
Somnath, Suhas committed
912
913
            udvs_table[BE_IF_switch == 1, 5] = udvs_table[BE_IF_switch == 1, 1]
            udvs_table[BE_OF_switch == 1, 6] = udvs_table[BE_IF_switch == 1, 1]
Unknown's avatar
Unknown committed
914

Somnath, Suhas's avatar
Somnath, Suhas committed
915
        elif VS_ACDC_cond is 2:  # AC voltage spectroscopy
Unknown's avatar
Unknown committed
916

Somnath, Suhas's avatar
Somnath, Suhas committed
917
            num_VS_steps = vs_amp_vec.size
Unknown's avatar
Unknown committed
918
919
            half = int(0.5 * num_VS_steps)

Somnath, Suhas's avatar
Somnath, Suhas committed
920
            if num_VS_steps is not half * 2:
921
                raise ValueError('Odd number of UDVS steps found. Exiting!')
Unknown's avatar
Unknown committed
922
923

            UD_dc_vec = VS_offset * np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
924
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'forward', 'reverse']
Somnath, Suhas's avatar
Somnath, Suhas committed
925
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'A', 'A']
Somnath, Suhas's avatar
Somnath, Suhas committed
926
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Unknown's avatar
Unknown committed
927
            udvs_table[:, 0] = np.arange(1, num_VS_steps + 1)
Somnath, Suhas's avatar
Somnath, Suhas committed
928
929
930
            udvs_table[:, 1] = UD_dc_vec
            udvs_table[:, 2] = vs_amp_vec
            udvs_table[:, 3] = np.ones(num_VS_steps)
Unknown's avatar
Unknown committed
931
932
933
934
            udvs_table[:half, 4] = BE_signal_type * np.ones(half)
            udvs_table[half:, 4] = -1 * BE_signal_type * np.ones(half)
            udvs_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN') * np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
935
936
            udvs_table[:half, 5] = vs_amp_vec[:half]
            udvs_table[half:, 6] = vs_amp_vec[half:]
Unknown's avatar
Unknown committed
937

Somnath, Suhas's avatar
Somnath, Suhas committed
938
939
        return UD_VS_table_label, UD_VS_table_unit, udvs_table

940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
    @staticmethod
    def __get_random_spectra(parsers, num_pixels, num_udvs_steps, num_bins, num_spectra=100, verbose=False):
        """
        Parameters
        ----------
        parsers : list of BEodfParser objects
            parsers to seek into files to grab spectra
        num_pixels : unsigned int
            Number of spatial positions in the image
        num_udvs_steps : unsigned int
            Number of UDVS steps
        num_bins : unsigned int
            Number of frequency bins in every UDVS step
        num_spectra : unsigned int
            Total number of spectra to be extracted
        verbose : Boolean, optional
            Whether or not to print debugging statements

        Returns
        -------
        chosen_spectra : 2D complex numpy array
            spectrogram or spectra arranged as [instance, spectrum]
        """
        num_pixels = int(num_pixels)
        num_udvs_steps = int(num_udvs_steps)
        num_bins = int(<