be_odf.py 46.8 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:24:12 2015

@author: Suhas Somnath, Stephen Jesse
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

Somnath, Suhas's avatar
Somnath, Suhas committed
10
from os import path, listdir, remove
11
from warnings import warn
12

Somnath, Suhas's avatar
Somnath, Suhas committed
13
14
import numpy as np
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15

16
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
17
    createSpecVals, requires_conjugate, nf32
Somnath, Suhas's avatar
Somnath, Suhas committed
18
from .translator import Translator
19
from .utils import generate_dummy_main_parms, build_ind_val_dsets
Somnath, Suhas's avatar
Somnath, Suhas committed
20
21
22
from ..hdf_utils import getH5DsetRefs, linkRefs, calc_chunks
from ..io_hdf5 import ioHDF5
from ..microdata import MicroDataGroup, MicroDataset
23

24

Somnath, Suhas's avatar
Somnath, Suhas committed
25
26
27
28
29
class BEodfTranslator(Translator):
    """
    Translates either the Band Excitation (BE) scan or Band Excitation 
    Polarization Switching (BEPS) data format from the old data format(s) to .h5
    """
Unknown's avatar
Unknown committed
30

Chris Smith's avatar
Chris Smith committed
31
32
33
34
35
    def __init__(self, *args, **kwargs):
        super(BEodfTranslator, self).__init__(*args, **kwargs)

        self.hdf = None
        self.h5_raw = None
36
        self.num_rand_spectra = kwargs.pop('num_rand_spectra', 1000)
Chris Smith's avatar
Chris Smith committed
37

38
    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False, verbose=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
39
40
41
42
43
44
45
46
47
48
49
50
51
52
        """
        Translates .dat data file(s) to a single .h5 file
        
        Parameters
        -------------
        file_path : String / Unicode
            Absolute file path for one of the data files. 
            It is assumed that this file is of the OLD data format.
        show_plots : (optional) Boolean
            Whether or not to show intermediate plots
        save_plots : (optional) Boolean
            Whether or not to save plots to disk
        do_histogram : (optional) Boolean
            Whether or not to construct histograms to visualize data quality. Note - this takes a fair amount of time
53
54
        verbose : (optional) Boolean
            Whether or not to print statements
Somnath, Suhas's avatar
Somnath, Suhas committed
55
56
57
58
59
60
61
            
        Returns
        ----------
        h5_path : String / Unicode
            Absolute path of the resultant .h5 file
        """
        (folder_path, basename) = path.split(file_path)
62
        (basename, path_dict) = self._parse_file_path(file_path)
Unknown's avatar
Unknown committed
63

Somnath, Suhas's avatar
Somnath, Suhas committed
64
        h5_path = path.join(folder_path, basename + '.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
65
66
        tot_bins_multiplier = 1
        udvs_denom = 2
Unknown's avatar
Unknown committed
67

Somnath, Suhas's avatar
Somnath, Suhas committed
68
        if 'parm_txt' in path_dict.keys():
Unknown's avatar
Unknown committed
69
            (isBEPS, parm_dict) = parmsToDict(path_dict['parm_txt'])
Somnath, Suhas's avatar
Somnath, Suhas committed
70
71
        elif 'old_mat_parms' in path_dict.keys():
            isBEPS = True
Somnath, Suhas's avatar
Somnath, Suhas committed
72
            parm_dict = self.__get_parms_from_old_mat(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
73
        else:
74
            raise IOError('No parameters file found! Cannot translate this dataset!')
Unknown's avatar
Unknown committed
75

Somnath, Suhas's avatar
Somnath, Suhas committed
76
77
78
        ignored_plt_grps = []
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'
Unknown's avatar
Unknown committed
79

Somnath, Suhas's avatar
Somnath, Suhas committed
80
81
            field_mode = parm_dict['VS_measure_in_field_loops']
            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'
Unknown's avatar
Unknown committed
82

Somnath, Suhas's avatar
Somnath, Suhas committed
83
            if not std_expt:
84
                raise ValueError('This translator does not handle user defined voltage spectroscopy')
Unknown's avatar
Unknown committed
85
86
87

            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode'])

Somnath, Suhas's avatar
Somnath, Suhas committed
88
            if parm_dict['VS_mode'] in ['DC modulation mode', 'current mode']:
Somnath, Suhas's avatar
Somnath, Suhas committed
89
90
91
92
93
94
95
96
97
98
99
                if field_mode == 'in and out-of-field':
                    tot_bins_multiplier = 2
                    udvs_denom = 1
                else:
                    if field_mode == 'out-of-field':
                        ignored_plt_grps = ['in-field']
                    else:
                        ignored_plt_grps = ['out-of-field']
            else:
                tot_bins_multiplier = 1
                udvs_denom = 1
Unknown's avatar
Unknown committed
100

Somnath, Suhas's avatar
Somnath, Suhas committed
101
102
103
        else:
            spec_label = 'None'
            parm_dict['data_type'] = 'BELineData'
Unknown's avatar
Unknown committed
104

Somnath, Suhas's avatar
Somnath, Suhas committed
105
106
        # Check file sizes:
        if 'read_real' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
107
108
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
Somnath, Suhas's avatar
Somnath, Suhas committed
109
110
111
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])
Unknown's avatar
Unknown committed
112

Somnath, Suhas's avatar
Somnath, Suhas committed
113
114
115
        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

Unknown's avatar
Unknown committed
116
        add_pix = False
Somnath, Suhas's avatar
Somnath, Suhas committed
117
118
        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
Unknown's avatar
Unknown committed
119
120
        num_pix = num_rows * num_cols
        tot_bins = real_size / (num_pix * 4)
Chris Smith's avatar
Chris Smith committed
121
        # Check for case where only a single pixel is missing.
Unknown's avatar
Unknown committed
122
123
124
        check_bins = real_size / ((num_pix - 1) * 4)

        if tot_bins % 1 and check_bins % 1:
125
            raise ValueError('Aborting! Some parameter appears to have changed in-between')
Somnath, Suhas's avatar
Somnath, Suhas committed
126
        elif not tot_bins % 1:
Chris Smith's avatar
Chris Smith committed
127
            # Everything's ok
Somnath, Suhas's avatar
Somnath, Suhas committed
128
129
130
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
Unknown's avatar
Unknown committed
131
132
133
134
135
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.')
            add_pix = True

        tot_bins = int(tot_bins) * tot_bins_multiplier

Somnath, Suhas's avatar
Somnath, Suhas committed
136
        if 'parm_mat' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
137
            (bin_inds, bin_freqs, bin_FFT, ex_wfm) = self.__read_parms_mat(path_dict['parm_mat'], isBEPS)
Somnath, Suhas's avatar
Somnath, Suhas committed
138
        elif 'old_mat_parms' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
139
            (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__read_old_mat_be_vecs(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
140
        else:
Unknown's avatar
Unknown committed
141
            band_width = parm_dict['BE_band_width_[Hz]'] * (0.5 - parm_dict['BE_band_edge_trim'])
Somnath, Suhas's avatar
Somnath, Suhas committed
142
            st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
Unknown's avatar
Unknown committed
143
            en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width
Somnath, Suhas's avatar
Somnath, Suhas committed
144
            bin_freqs = np.linspace(st_f, en_f, tot_bins, dtype=np.float32)
Unknown's avatar
Unknown committed
145

146
            warn('No parms .mat file found.... Filling dummy values into ancillary datasets.')
Somnath, Suhas's avatar
Somnath, Suhas committed
147
148
149
            bin_inds = np.zeros(shape=tot_bins, dtype=np.int32)
            bin_FFT = np.zeros(shape=tot_bins, dtype=np.complex64)
            ex_wfm = np.zeros(shape=100, dtype=np.float32)
Unknown's avatar
Unknown committed
150

Somnath, Suhas's avatar
Somnath, Suhas committed
151
152
153
154
155
        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)
156
157
158

        ds_ex_wfm = MicroDataset('Excitation_Waveform', ex_wfm)

Somnath, Suhas's avatar
Somnath, Suhas committed
159
        self.FFT_BE_wave = bin_FFT
160

161
        ds_pos_ind, ds_pos_val = build_ind_val_dsets([num_cols, num_rows], is_spectral=False,
162
                                                     labels=['X', 'Y'], units=['m', 'm'], verbose=verbose)
Unknown's avatar
Unknown committed
163

Somnath, Suhas's avatar
Somnath, Suhas committed
164
        if isBEPS:
Somnath, Suhas's avatar
Somnath, Suhas committed
165
            (UDVS_labs, UDVS_units, UDVS_mat) = self.__build_udvs_table(parm_dict)
Unknown's avatar
Unknown committed
166
167

            #             Remove the unused plot group columns before proceeding:
Somnath, Suhas's avatar
Somnath, Suhas committed
168
            (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
Unknown's avatar
Unknown committed
169

Somnath, Suhas's avatar
Somnath, Suhas committed
170
            spec_inds = np.zeros(shape=(2, tot_bins), dtype=np.uint)
Unknown's avatar
Unknown committed
171
172
173
174
175

            #             Will assume that all excitation waveforms have same number of bins
            num_actual_udvs_steps = UDVS_mat.shape[0] / udvs_denom
            bins_per_step = tot_bins / num_actual_udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
176
            if bins_per_step % 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
177
178
                print('UDVS mat shape: {}, total bins: {}, bins per step: {}'.format(UDVS_mat.shape, tot_bins,
                                                                                     bins_per_step))
179
                raise ValueError('Non integer number of bins per step!')
Unknown's avatar
Unknown committed
180

Somnath, Suhas's avatar
Somnath, Suhas committed
181
182
            bins_per_step = int(bins_per_step)
            num_actual_udvs_steps = int(num_actual_udvs_steps)
Unknown's avatar
Unknown committed
183
184
185

            stind = 0
            for step_index in range(UDVS_mat.shape[0]):
Unknown's avatar
Unknown committed
186
187
188
                if UDVS_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                    continue
                # Bin step
Unknown's avatar
Unknown committed
189
                spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=np.uint32)
Unknown's avatar
Unknown committed
190
                # UDVS step
Unknown's avatar
Unknown committed
191
                spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step, dtype=np.uint32)
Somnath, Suhas's avatar
Somnath, Suhas committed
192
                stind += bins_per_step
Somnath, Suhas's avatar
Somnath, Suhas committed
193
            del stind, step_index
Unknown's avatar
Unknown committed
194

Somnath, Suhas's avatar
Somnath, Suhas committed
195
        else:  # BE Line
Somnath, Suhas's avatar
Somnath, Suhas committed
196
            self.signal_type = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
197
            self.expt_type = 1  # Stephen has not used this index for some reason
Somnath, Suhas's avatar
Somnath, Suhas committed
198
199
            num_actual_udvs_steps = 1
            bins_per_step = tot_bins
Somnath, Suhas's avatar
Somnath, Suhas committed
200
            UDVS_labs = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'be-line']
Somnath, Suhas's avatar
Somnath, Suhas committed
201
            UDVS_units = ['', 'V', 'A', '', '', '']
Somnath, Suhas's avatar
Somnath, Suhas committed
202
203
            UDVS_mat = np.array([1, 0, parm_dict['BE_amplitude_[V]'], 1, 1, 1],
                                dtype=np.float32).reshape(1, len(UDVS_labs))
Somnath, Suhas's avatar
Somnath, Suhas committed
204
205

            spec_inds = np.vstack((np.arange(tot_bins, dtype=np.uint), np.zeros(tot_bins, dtype=np.uint32)))
Unknown's avatar
Unknown committed
206

Somnath, Suhas's avatar
Somnath, Suhas committed
207
208
209
        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
210
        parm_dict['num_udvs_steps'] = num_actual_udvs_steps
Unknown's avatar
Unknown committed
211

Somnath, Suhas's avatar
Somnath, Suhas committed
212
        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
213
        for col_ind, col_name in enumerate(UDVS_labs):
Unknown's avatar
Unknown committed
214
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind + 1))
Somnath, Suhas's avatar
Somnath, Suhas committed
215
216
217
        ds_UDVS = MicroDataset('UDVS', UDVS_mat)
        ds_UDVS.attrs['labels'] = udvs_slices
        ds_UDVS.attrs['units'] = UDVS_units
Unknown's avatar
Unknown committed
218
219
220
221
        #         ds_udvs_labs = MicroDataset('UDVS_Labels',np.array(UDVS_labs))
        ds_UDVS_inds = MicroDataset('UDVS_Indices', spec_inds[1])

        #         ds_spec_labs = MicroDataset('Spectroscopic_Labels',np.array(['Bin','UDVS_Step']))
Somnath, Suhas's avatar
Somnath, Suhas committed
222
        ds_bin_steps = MicroDataset('Bin_Step', np.arange(bins_per_step, dtype=np.uint32), dtype=np.uint32)
Unknown's avatar
Unknown committed
223

Somnath, Suhas's avatar
Somnath, Suhas committed
224
        # Need to add the Bin Waveform type - infer from UDVS        
Unknown's avatar
Unknown committed
225
        exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
Somnath, Suhas's avatar
Somnath, Suhas committed
226
227
228

        if self.expt_type == 2:
            # Need to double the vectors:
Unknown's avatar
Unknown committed
229
            exec_bin_vec = np.hstack((exec_bin_vec, -1 * exec_bin_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
230
231
            bin_inds = np.hstack((bin_inds, bin_inds))
            bin_freqs = np.hstack((bin_freqs, bin_freqs))
Somnath, Suhas's avatar
Somnath, Suhas committed
232
            # This is wrong but I don't know what else to do
Somnath, Suhas's avatar
Somnath, Suhas committed
233
            bin_FFT = np.hstack((bin_FFT, bin_FFT))
Unknown's avatar
Unknown committed
234
235
236

        ds_bin_inds = MicroDataset('Bin_Indices', bin_inds, dtype=np.uint32)
        ds_bin_freq = MicroDataset('Bin_Frequencies', bin_freqs)
Somnath, Suhas's avatar
Somnath, Suhas committed
237
238
        ds_bin_FFT = MicroDataset('Bin_FFT', bin_FFT)
        ds_wfm_typ = MicroDataset('Bin_Wfm_Type', exec_bin_vec)
Unknown's avatar
Unknown committed
239

Somnath, Suhas's avatar
Somnath, Suhas committed
240
        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
Somnath, Suhas's avatar
Somnath, Suhas committed
241
242
243
244
245
246
247
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_labs_names = createSpecVals(UDVS_mat,
                                                                                                     spec_inds,
                                                                                                     bin_freqs,
                                                                                                     exec_bin_vec,
                                                                                                     parm_dict,
                                                                                                     UDVS_labs,
                                                                                                     UDVS_units)
Chris Smith's avatar
Chris Smith committed
248

Somnath, Suhas's avatar
Somnath, Suhas committed
249
        spec_vals_slices = dict()
Unknown's avatar
Unknown committed
250
251
252
        #         if len(spec_vals_labs) == 1:
        #             spec_vals_slices[spec_vals_labs[0]]=(slice(0,1,None),)
        #         else:
Somnath, Suhas's avatar
Somnath, Suhas committed
253
254

        for row_ind, row_name in enumerate(spec_vals_labs):
Unknown's avatar
Unknown committed
255
            spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
Somnath, Suhas's avatar
Somnath, Suhas committed
256
257
258

        ds_spec_mat = MicroDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
        ds_spec_mat.attrs['labels'] = spec_vals_slices
Unknown's avatar
Unknown committed
259
        ds_spec_mat.attrs['units'] = spec_vals_units
Somnath, Suhas's avatar
Somnath, Suhas committed
260
        ds_spec_vals_mat = MicroDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
Somnath, Suhas's avatar
Somnath, Suhas committed
261
262
263
        ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
        ds_spec_vals_mat.attrs['units'] = spec_vals_units
        for entry in spec_vals_labs_names:
Unknown's avatar
Unknown committed
264
            label = entry[0] + '_parameters'
Somnath, Suhas's avatar
Somnath, Suhas committed
265
            names = entry[1]
Somnath, Suhas's avatar
Somnath, Suhas committed
266
267
            ds_spec_mat.attrs[label] = names
            ds_spec_vals_mat.attrs[label] = names
Chris Smith's avatar
Chris Smith committed
268

Somnath, Suhas's avatar
Somnath, Suhas committed
269
        # Noise floor should be of shape: (udvs_steps x 3 x positions)
Somnath, Suhas's avatar
Somnath, Suhas committed
270
271
        ds_noise_floor = MicroDataset('Noise_Floor', np.zeros(shape=(num_pix, num_actual_udvs_steps), dtype=nf32),
                                      chunking=(1, num_actual_udvs_steps))
Somnath, Suhas's avatar
Somnath, Suhas committed
272
273

        """
Chris Smith's avatar
Chris Smith committed
274
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares
Somnath, Suhas's avatar
Somnath, Suhas committed
275
        of UDVS steps by pixels.  N is determined dynamically based on the dimensions of the
Chris Smith's avatar
Chris Smith committed
276
        dataset.  Currently it is set such that individual chunks are less than 10kB in size.
Somnath, Suhas's avatar
Somnath, Suhas committed
277
278
279
        
        Chris Smith -- csmith55@utk.edu
        """
Chris Smith's avatar
Chris Smith committed
280
281
282
283
284
285
286
287
        BEPS_chunks = calc_chunks([num_pix, tot_bins],
                                  np.complex64(0).itemsize,
                                  unit_chunks=(1, bins_per_step))
        ds_main_data = MicroDataset('Raw_Data', data=[],
                                    maxshape=(num_pix, tot_bins),
                                    dtype=np.complex64,
                                    chunking=BEPS_chunks,
                                    compression='gzip')
Unknown's avatar
Unknown committed
288

Somnath, Suhas's avatar
Somnath, Suhas committed
289
290
291
292
        chan_grp = MicroDataGroup('Channel_')
        chan_grp.attrs['Channel_Input'] = parm_dict['IO_Analog_Input_1']
        chan_grp.addChildren([ds_main_data, ds_noise_floor])
        chan_grp.addChildren([ds_ex_wfm, ds_pos_ind, ds_pos_val, ds_spec_mat, ds_UDVS,
Chris Smith's avatar
Chris Smith committed
293
294
                              ds_bin_steps, ds_bin_inds, ds_bin_freq, ds_bin_FFT,
                              ds_wfm_typ, ds_spec_vals_mat, ds_UDVS_inds])
Unknown's avatar
Unknown committed
295

Somnath, Suhas's avatar
Somnath, Suhas committed
296
297
298
299
        # technically should change the date, etc.
        meas_grp = MicroDataGroup('Measurement_')
        meas_grp.attrs = parm_dict
        meas_grp.addChildren([chan_grp])
Unknown's avatar
Unknown committed
300

Somnath, Suhas's avatar
Somnath, Suhas committed
301
        spm_data = MicroDataGroup('')
302
        global_parms = generate_dummy_main_parms()
Somnath, Suhas's avatar
Somnath, Suhas committed
303
304
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
305
306
307
308
        try:
            global_parms['experiment_date'] = parm_dict['File_date_and_time']
        except KeyError:
            global_parms['experiment_date'] = '1:1:1'
Chris Smith's avatar
Chris Smith committed
309

Somnath, Suhas's avatar
Somnath, Suhas committed
310
        # assuming that the experiment was completed:
Unknown's avatar
Unknown committed
311
312
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
Somnath, Suhas's avatar
Somnath, Suhas committed
313
        global_parms['data_type'] = parm_dict['data_type']
Somnath, Suhas's avatar
Somnath, Suhas committed
314
        global_parms['translator'] = 'ODF'
Unknown's avatar
Unknown committed
315

Somnath, Suhas's avatar
Somnath, Suhas committed
316
317
        spm_data.attrs = global_parms
        spm_data.addChildren([meas_grp])
Unknown's avatar
Unknown committed
318

Somnath, Suhas's avatar
Somnath, Suhas committed
319
320
        if path.exists(h5_path):
            remove(h5_path)
Unknown's avatar
Unknown committed
321

Somnath, Suhas's avatar
Somnath, Suhas committed
322
323
        # Write everything except for the main data.
        self.hdf = ioHDF5(h5_path)
Unknown's avatar
Unknown committed
324

325
        h5_refs = self.hdf.writeData(spm_data, print_log=verbose)
Unknown's avatar
Unknown committed
326

Somnath, Suhas's avatar
Somnath, Suhas committed
327
        self.h5_raw = getH5DsetRefs(['Raw_Data'], h5_refs)[0]
Unknown's avatar
Unknown committed
328

Somnath, Suhas's avatar
Somnath, Suhas committed
329
330
331
332
        # Now doing linkrefs:
        aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values',
                        'Spectroscopic_Indices', 'UDVS', 'Bin_Step', 'Bin_Indices', 'UDVS_Indices',
                        'Bin_Frequencies', 'Bin_FFT', 'Bin_Wfm_Type', 'Noise_Floor', 'Spectroscopic_Values']
333
        linkRefs(self.h5_raw, getH5DsetRefs(aux_ds_names, h5_refs))
Somnath, Suhas's avatar
Somnath, Suhas committed
334

Chris Smith's avatar
Chris Smith committed
335
        self._read_data(UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix)
Unknown's avatar
Unknown committed
336

Somnath, Suhas's avatar
Somnath, Suhas committed
337
338
        generatePlotGroups(self.h5_raw, self.hdf, self.mean_resp, folder_path, basename,
                           self.max_resp, self.min_resp, max_mem_mb=self.max_ram,
Somnath, Suhas's avatar
Somnath, Suhas committed
339
                           spec_label=spec_label, show_plots=show_plots, save_plots=save_plots,
Unknown's avatar
Unknown committed
340
                           do_histogram=do_histogram, debug=verbose)
Unknown's avatar
Unknown committed
341

Somnath, Suhas's avatar
Somnath, Suhas committed
342
        self.hdf.close()
Unknown's avatar
Unknown committed
343

Somnath, Suhas's avatar
Somnath, Suhas committed
344
        return h5_path
Chris Smith's avatar
Chris Smith committed
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372

    def _read_data(self, UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix):
        """
        Checks if the data is BEPS or BELine and calls the correct function to read the data from
        file

        Parameters
        ----------
        UDVS_mat : numpy.ndarray of float
            UDVS table
        parm_dict : dict
            Experimental parameters
        path_dict : dict
            Dictionary of data files to be read
        real_size : dict
            Size of each data file
        isBEPS : boolean
            Is the data BEPS
        add_pix : boolean
            Does the reader need to add extra pixels to the end of the dataset

        Returns
        -------
        None
        """
        # Now read the raw data files:
        if not isBEPS:
            # Do this for all BE-Line (always small enough to read in one shot)
373
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
374
375
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'out-of-field':
            # Do this for out-of-field BEPS ONLY that is also small (256 MB)
376
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
377
378
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'in-field':
            # Do this for in-field only
379
            self.__quick_read_data(path_dict['write_real'], path_dict['write_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
380
381
        else:
            # Large BEPS datasets OR those with in-and-out of field
Somnath, Suhas's avatar
Somnath, Suhas committed
382
            self.__read_beps_data(path_dict, UDVS_mat.shape[0], parm_dict['VS_measure_in_field_loops'], add_pix)
Chris Smith's avatar
Chris Smith committed
383
384
        self.hdf.file.flush()

Somnath, Suhas's avatar
Somnath, Suhas committed
385
    def __read_beps_data(self, path_dict, udvs_steps, mode, add_pixel=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
        """
        Reads the imaginary and real data files pixelwise and writes to the H5 file 
        
        Parameters 
        --------------------
        path_dict : dictionary
            Dictionary containing the absolute paths of the real and imaginary data files
        udvs_steps : unsigned int
            Number of UDVS steps
        mode : String / Unicode
            'in-field', 'out-of-field', or 'in and out-of-field'
        add_pixel : boolean. (Optional; default is False)
            If an empty pixel worth of data should be written to the end             
        
        Returns 
        -------------------- 
        None
        """
Unknown's avatar
Unknown committed
404

Somnath, Suhas's avatar
Somnath, Suhas committed
405
        print('---- reading pixel-by-pixel ----------')
Unknown's avatar
Unknown committed
406
407
408
409

        bytes_per_pix = self.h5_raw.shape[1] * 4
        step_size = self.h5_raw.shape[1] / udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
410
        if mode == 'out-of-field':
Unknown's avatar
Unknown committed
411
            parsers = [BEodfParser(path_dict['read_real'], path_dict['read_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
412
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
413
        elif mode == 'in-field':
Unknown's avatar
Unknown committed
414
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
415
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
416
417
        elif mode == 'in and out-of-field':
            # each file will only have half the udvs steps:
Unknown's avatar
Unknown committed
418
            if 0.5 * udvs_steps % 1:
419
420
                raise ValueError('Odd number of UDVS')

Unknown's avatar
Unknown committed
421
            udvs_steps = int(0.5 * udvs_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
422
            # be careful - each pair contains only half the necessary bins - so read half
Unknown's avatar
Unknown committed
423
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
424
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2)),
Unknown's avatar
Unknown committed
425
426
427
                       BEodfParser(path_dict['read_real'], path_dict['read_imag'],
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2))]

Somnath, Suhas's avatar
Somnath, Suhas committed
428
            if step_size % 1:
429
430
                raise ValueError('strange number of bins per UDVS step. Exiting')

Somnath, Suhas's avatar
Somnath, Suhas committed
431
            step_size = int(step_size)
432

433
434
        rand_spectra = self.__get_random_spectra(parsers, self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
435
        take_conjugate = requires_conjugate(rand_spectra)
436

Somnath, Suhas's avatar
Somnath, Suhas committed
437
438
439
440
        self.mean_resp = np.zeros(shape=(self.h5_raw.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)

Unknown's avatar
Unknown committed
441
        numpix = self.h5_raw.shape[0]
Somnath, Suhas's avatar
Somnath, Suhas committed
442
443
444
        """ 
        Don't try to do the last step if a pixel is missing.   
        This will be handled after the loop. 
Unknown's avatar
Unknown committed
445
446
447
448
        """
        if add_pixel:
            numpix -= 1

Somnath, Suhas's avatar
Somnath, Suhas committed
449
        for pix_indx in range(numpix):
Somnath, Suhas's avatar
Somnath, Suhas committed
450
            if self.h5_raw.shape[0] > 5:
Unknown's avatar
Unknown committed
451
452
453
                if pix_indx % int(round(self.h5_raw.shape[0] / 10)) == 0:
                    print('Reading... {} complete'.format(round(100 * pix_indx / self.h5_raw.shape[0])))

Somnath, Suhas's avatar
Somnath, Suhas committed
454
455
456
            # get the raw stream from each parser
            pxl_data = list()
            for prsr in parsers:
Somnath, Suhas's avatar
Somnath, Suhas committed
457
                pxl_data.append(prsr.read_pixel())
Unknown's avatar
Unknown committed
458

Somnath, Suhas's avatar
Somnath, Suhas committed
459
460
461
462
463
            # interleave if both in and out of field
            # we are ignoring user defined possibilities...
            if mode == 'in and out-of-field':
                in_fld = pxl_data[0]
                out_fld = pxl_data[1]
Unknown's avatar
Unknown committed
464

Somnath, Suhas's avatar
Somnath, Suhas committed
465
466
                in_fld_2 = in_fld.reshape(udvs_steps, step_size)
                out_fld_2 = out_fld.reshape(udvs_steps, step_size)
Unknown's avatar
Unknown committed
467
                raw_mat = np.empty((udvs_steps * 2, step_size), dtype=out_fld.dtype)
Somnath, Suhas's avatar
Somnath, Suhas committed
468
469
                raw_mat[0::2, :] = in_fld_2
                raw_mat[1::2, :] = out_fld_2
Somnath, Suhas's avatar
Somnath, Suhas committed
470
471
                raw_vec = raw_mat.reshape(in_fld.size + out_fld.size).transpose()
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
472
                raw_vec = pxl_data[0]  # only one parser
Somnath, Suhas's avatar
Somnath, Suhas committed
473
474
            self.max_resp[pix_indx] = np.max(np.abs(raw_vec))
            self.min_resp[pix_indx] = np.min(np.abs(raw_vec))
Unknown's avatar
Unknown committed
475
            self.mean_resp = (1 / (pix_indx + 1)) * (raw_vec + pix_indx * self.mean_resp)
476
477
478

            if take_conjugate:
                raw_vec = np.conjugate(raw_vec)
479
            self.h5_raw[pix_indx, :] = np.complex64(raw_vec[:])
Somnath, Suhas's avatar
Somnath, Suhas committed
480
            self.hdf.file.flush()
Unknown's avatar
Unknown committed
481

Somnath, Suhas's avatar
Somnath, Suhas committed
482
        # Add zeros to main_data for the missing pixel. 
Unknown's avatar
Unknown committed
483
484
485
        if add_pixel:
            self.h5_raw[-1, :] = 0 + 0j

Somnath, Suhas's avatar
Somnath, Suhas committed
486
        print('---- Finished reading files -----')
487
488

    def __quick_read_data(self, real_path, imag_path, udvs_steps):
Somnath, Suhas's avatar
Somnath, Suhas committed
489
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
490
491
492
493
494
495
496
497
        Returns information about the excitation BE waveform present in the .mat file

        Parameters
        -----------
        real_path : String / Unicode
            Absolute file path of the real data file
        imag_path : String / Unicode
            Absolute file path of the real data file
498
499
        udvs_steps : unsigned int
            Number of UDVS steps
Somnath, Suhas's avatar
Somnath, Suhas committed
500
        """
Unknown's avatar
Unknown committed
501
        print('---- reading all data at once ----------')
Somnath, Suhas's avatar
Somnath, Suhas committed
502

Unknown's avatar
Unknown committed
503
        parser = BEodfParser(real_path, imag_path, self.h5_raw.shape[0], self.h5_raw.shape[1] * 4)
504
505

        step_size = self.h5_raw.shape[1] / udvs_steps
506
507
        rand_spectra = self.__get_random_spectra([parser], self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
508
        take_conjugate = requires_conjugate(rand_spectra)
Somnath, Suhas's avatar
Somnath, Suhas committed
509
        raw_vec = parser.read_all_data()
510
        if take_conjugate:
511
            print('Taking conjugate to ensure positive Quality factors')
512
            raw_vec = np.conjugate(raw_vec)
Unknown's avatar
Unknown committed
513

Somnath, Suhas's avatar
Somnath, Suhas committed
514
        raw_mat = raw_vec.reshape(self.h5_raw.shape[0], self.h5_raw.shape[1])
Unknown's avatar
Unknown committed
515

Somnath, Suhas's avatar
Somnath, Suhas committed
516
        # Write to the h5 dataset:
Somnath, Suhas's avatar
Somnath, Suhas committed
517
518
519
        self.mean_resp = np.mean(raw_mat, axis=0)
        self.max_resp = np.amax(np.abs(raw_mat), axis=0)
        self.min_resp = np.amin(np.abs(raw_mat), axis=0)
520
        self.h5_raw[:, :] = np.complex64(raw_mat)
Somnath, Suhas's avatar
Somnath, Suhas committed
521
522
        self.hdf.file.flush()

Unknown's avatar
Unknown committed
523
524
        print('---- Finished reading files -----')

525
    def _parse_file_path(self, data_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
526
527
528
529
530
531
532
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary
        
        Parameters 
        --------------------
        data_filepath: String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
533
            Absolute path of any file in the same directory as the .dat files
Somnath, Suhas's avatar
Somnath, Suhas committed
534
535
536
537
538
539
540
541
542
        
        Returns 
        --------------------
        basename : String / Unicode
            Basename of the dataset      
        path_dict : Dictionary
            Dictionary containing absolute paths of all necessary data and parameter files
        """
        (folder_path, basename) = path.split(data_filepath)
Unknown's avatar
Unknown committed
543
        (super_folder, basename) = path.split(folder_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
544
545
546
547
548
549
550
551
552
553
554

        if basename.endswith('_d'):
            # Old old data format where the folder ended with a _d for some reason
            basename = basename[:-2]
        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()
Unknown's avatar
Unknown committed
555

Somnath, Suhas's avatar
Somnath, Suhas committed
556
        for file_name in listdir(folder_path):
Chris Smith's avatar
Chris Smith committed
557
            abs_path = path.join(folder_path, file_name)
Somnath, Suhas's avatar
Somnath, Suhas committed
558
559
560
561
562
            if file_name.endswith('.txt') and file_name.find('parm') > 0:
                path_dict['parm_txt'] = abs_path
            elif file_name.find('.mat') > 0:
                if file_name.find('more_parms') > 0:
                    path_dict['parm_mat'] = abs_path
Unknown's avatar
Unknown committed
563
                elif file_name == (basename + '.mat'):
Somnath, Suhas's avatar
Somnath, Suhas committed
564
565
566
567
568
569
570
571
572
573
574
575
                    path_dict['old_mat_parms'] = abs_path
            elif file_name.endswith('.dat'):
                # Need to account for the second AI channel here
                file_tag = 'read'
                if file_name.find('write') > 0:
                    file_tag = 'write'
                if file_name.find('real') > 0:
                    file_tag += '_real'
                elif file_name.find('imag') > 0:
                    file_tag += '_imag'
                path_dict[file_tag] = abs_path

Chris Smith's avatar
Chris Smith committed
576
        return basename, path_dict
Somnath, Suhas's avatar
Somnath, Suhas committed
577
578
579

    @staticmethod
    def __read_old_mat_be_vecs(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
        """
        Returns information about the excitation BE waveform present in the 
        more parms.mat file
        
        Parameters 
        --------------------
        filepath : String or unicode
            Absolute filepath of the .mat parameter file
        
        Returns 
        --------------------
        bin_inds : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        bin_w : 1D numpy float array
            Excitation bin Frequencies
        bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        BE_wave : 1D numpy float array
            Band Excitation waveform
        dc_amp_vec_full : 1D numpy float array
            spectroscopic waveform. 
            This information will be necessary for fixing the UDVS for AC modulation for example
        """
Unknown's avatar
Unknown committed
603
        matread = loadmat(file_path, squeeze_me=True)
Somnath, Suhas's avatar
Somnath, Suhas committed
604
        BE_wave = matread['BE_wave']
Unknown's avatar
Unknown committed
605
        bin_inds = matread['bin_ind'] - 1  # Python base 0
Somnath, Suhas's avatar
Somnath, Suhas committed
606
607
608
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
Somnath, Suhas's avatar
Somnath, Suhas committed
609
610
        bin_FFT = np.conjugate(FFT_full[bin_inds])
        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full
Unknown's avatar
Unknown committed
611

Somnath, Suhas's avatar
Somnath, Suhas committed
612
613
    @staticmethod
    def __get_parms_from_old_mat(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it
        
        Parameters 
        --------------------
        file_path : Unicode / String
            absolute filepath of the .mat file containing the parameters
            
        Returns 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)
Unknown's avatar
Unknown committed
630
631
632

        parm_dict['IO_rate'] = str(int(matread['AO_rate'] / 1E+6)) + ' MHz'

Somnath, Suhas's avatar
Somnath, Suhas committed
633
634
635
636
637
        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = position_vec[2]
        parm_dict['grid_num_cols'] = position_vec[3]
Unknown's avatar
Unknown committed
638

Somnath, Suhas's avatar
Somnath, Suhas committed
639
640
        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
Somnath, Suhas's avatar
Somnath, Suhas committed
641
            parm_dict['grid_num_rows'] = position_vec[0]  # set to number of present cols and rows
Somnath, Suhas's avatar
Somnath, Suhas committed
642
            parm_dict['grid_num_cols'] = position_vec[1]
Unknown's avatar
Unknown committed
643

Somnath, Suhas's avatar
Somnath, Suhas committed
644
645
646
647
648
649
650
651
652
        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
653
654
        parm_dict['BE_band_edge_smoothing_[s]'] = BE_parm_vec_1[4]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
Unknown's avatar
Unknown committed
655
656
657
        parm_dict['BE_window_adjustment'] = BE_parm_vec_1[6]
        parm_dict['BE_points_per_step'] = 2 ** int(BE_parm_vec_1[7])
        parm_dict['BE_repeats'] = 2 ** int(BE_parm_vec_1[8])
Somnath, Suhas's avatar
Somnath, Suhas committed
658
659
660
661
        try:
            parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        except KeyError:
            parm_dict['BE_bins_per_read'] = len(matread['bin_w'])
Unknown's avatar
Unknown committed
662

Somnath, Suhas's avatar
Somnath, Suhas committed
663
        assembly_parm_vec = matread['assembly_parm_vec']
Unknown's avatar
Unknown committed
664

Somnath, Suhas's avatar
Somnath, Suhas committed
665
666
667
668
669
670
        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
Unknown's avatar
Unknown committed
671

Somnath, Suhas's avatar
Somnath, Suhas committed
672
673
674
675
676
        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'
Unknown's avatar
Unknown committed
677

Somnath, Suhas's avatar
Somnath, Suhas committed
678
679
        # num_driving_bands = assembly_parm_vec[0]  # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1]  # 0 parallel 1 series
Unknown's avatar
Unknown committed
680

Somnath, Suhas's avatar
Somnath, Suhas committed
681
682
        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']
Unknown's avatar
Unknown committed
683
684
685
686

        VS_start_V = VS_parms[4]
        VS_start_loop_amp = VS_parms[5]
        VS_final_loop_amp = VS_parms[6]
Somnath, Suhas's avatar
Somnath, Suhas committed
687
        # VS_read_write_ratio = VS_parms[8]  # 1 <- SS_read_write_ratio
Unknown's avatar
Unknown committed
688

Somnath, Suhas's avatar
Somnath, Suhas committed
689
        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
Unknown's avatar
Unknown committed
690
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
691
692
        parm_dict['VS_steps_per_full_cycle'] = VS_parms[7]
        parm_dict['VS_cycle_fraction'] = 'full'
Unknown's avatar
Unknown committed
693
        parm_dict['VS_cycle_phase_shift'] = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
694
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
Somnath, Suhas's avatar
Somnath, Suhas committed
695
696
697
698
699
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0
Unknown's avatar
Unknown committed
700

Somnath, Suhas's avatar
Somnath, Suhas committed
701
702
        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
Unknown's avatar
Unknown committed
703
704
705
            parm_dict['VS_amplitude_[V]'] = 0.5 * (
                max(dc_amp_vec_full) - min(dc_amp_vec_full))  # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)
Somnath, Suhas's avatar
Somnath, Suhas committed
706
707
708
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
Somnath, Suhas's avatar
Somnath, Suhas committed
709
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
710
            parm_dict['VS_offset_[V]'] = 0
Unknown's avatar
Unknown committed
711
            parm_dict['VS_number_of_cycles'] = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
712
713
714
715
716
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
717
718
719
        elif VS_parms[0] == 2:
            # AC mode 
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
Somnath, Suhas's avatar
Somnath, Suhas committed
720
721
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
            parm_dict['VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
Somnath, Suhas's avatar
Somnath, Suhas committed
722
723
        else:
            parm_dict['VS_mode'] = 'Custom'
Unknown's avatar
Unknown committed
724

Somnath, Suhas's avatar
Somnath, Suhas committed
725
        return parm_dict
Unknown's avatar
Unknown committed
726

Somnath, Suhas's avatar
Somnath, Suhas committed
727
728
    @staticmethod
    def __read_parms_mat(file_path, is_beps):
Somnath, Suhas's avatar
Somnath, Suhas committed
729
730
731
732
733
        """
        Returns information about the excitation BE waveform present in the more parms.mat file
        
        Parameters 
        --------------------
Somnath, Suhas's avatar
Somnath, Suhas committed
734
        file_path : String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
735
            Absolute filepath of the .mat parameter file
Somnath, Suhas's avatar
Somnath, Suhas committed
736
        is_beps : Boolean
Somnath, Suhas's avatar
Somnath, Suhas committed
737
738
739
740
741
742
743
744
745
746
747
748
749
            Whether or not this is BEPS or BE-Line
        
        Returns 
        --------------------
        BE_bin_ind : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        BE_bin_w : 1D numpy float array
            Excitation bin Frequencies
        BE_bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        ex_wfm : 1D numpy float array
            Band Excitation waveform
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
750
        if not path.exists(file_path):
751
            raise IOError('NO "More parms" file found')
Somnath, Suhas's avatar
Somnath, Suhas committed
752
        if is_beps:
Somnath, Suhas's avatar
Somnath, Suhas committed
753
754
755
            fft_name = 'FFT_BE_wave'
        else:
            fft_name = 'FFT_BE_rev_wave'
Somnath, Suhas's avatar
Somnath, Suhas committed
756
        matread = loadmat(file_path, variable_names=['BE_bin_ind', 'BE_bin_w', fft_name])
Unknown's avatar
Unknown committed
757
        BE_bin_ind = np.squeeze(matread['BE_bin_ind']) - 1  # From Matlab (base 1) to Python (base 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
758
759
        BE_bin_w = np.squeeze(matread['BE_bin_w'])
        FFT_full = np.complex64(np.squeeze(matread[fft_name]))
Somnath, Suhas's avatar
Somnath, Suhas committed
760
        # For whatever weird reason, the sign of the imaginary portion is flipped. Correct it:
Unknown's avatar
Unknown committed
761
        # BE_bin_FFT = np.conjugate(FFT_full[BE_bin_ind])
Somnath, Suhas's avatar
Somnath, Suhas committed
762
763
        BE_bin_FFT = np.zeros(len(BE_bin_ind), dtype=np.complex64)
        BE_bin_FFT.real = np.real(FFT_full[BE_bin_ind])
Unknown's avatar
Unknown committed
764
765
        BE_bin_FFT.imag = -1 * np.imag(FFT_full[BE_bin_ind])

Somnath, Suhas's avatar
Somnath, Suhas committed
766
        ex_wfm = np.real(np.fft.ifft(np.fft.ifftshift(FFT_full)))
Somnath, Suhas's avatar
Somnath, Suhas committed
767
768

        return BE_bin_ind, BE_bin_w, BE_bin_FFT, ex_wfm
Unknown's avatar
Unknown committed
769

Somnath, Suhas's avatar
Somnath, Suhas committed
770
    def __build_udvs_table(self, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
        """
        Generates the UDVS table using the parameters
        
        Parameters 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        
        Returns 
        --------------------      
        UD_VS_table_label : List of strings
            Labels for columns in the UDVS table
        UD_VS_table_unit : List of strings
            Units for the columns in the UDVS table
        UD_VS_table : 2D numpy float array
            UDVS data table
        """
Unknown's avatar
Unknown committed
788

Somnath, Suhas's avatar
Somnath, Suhas committed
789
        def translate_val(target, strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
790
791
            """
            Internal function - Interprets the provided value using the provided lookup table
Somnath, Suhas's avatar
Somnath, Suhas committed
792
793
794
795
796
797
798
799
800

            Parameters
            ----------
            target : String
                Item we are looking for in the strvals list
            strvals : list of strings
                List of source values
            numvals : list of numbers
                List of results
Somnath, Suhas's avatar
Somnath, Suhas committed
801
            """
Unknown's avatar
Unknown committed
802

Somnath, Suhas's avatar
Somnath, Suhas committed
803
            if len(strvals) is not len(numvals):
Unknown's avatar
Unknown committed
804
                return None
Somnath, Suhas's avatar
Somnath, Suhas committed
805
            for strval, fltval in zip(strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
806
807
                if target == strval:
                    return fltval
Somnath, Suhas's avatar
Somnath, Suhas committed
808
            return None  # not found in list
Unknown's avatar
Unknown committed
809
810

        # % Extract values from parm text file
Unknown's avatar
Unknown committed
811
        BE_signal_type = translate_val(parm_dict['BE_phase_content'],
Unknown's avatar
Unknown committed
812
813
814
                                       ['chirp-sinc hybrid', '1/2 harmonic excitation',
                                        '1/3 harmonic excitation', 'pure sine'],
                                       [1, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
815
816
817
818
819
820
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type
        if BE_signal_type is 4:
            self.harmonic = 1
        BE_amp = parm_dict['BE_amplitude_[V]']
Unknown's avatar
Unknown committed
821

Somnath, Suhas's avatar
Somnath, Suhas committed
822
823
        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
Unknown's avatar
Unknown committed
824
        # VS_read_voltage = parm_dict['VS_read_voltage_[V]']
Somnath, Suhas's avatar
Somnath, Suhas committed
825
826
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
Somnath, Suhas's avatar
Somnath, Suhas committed
827
828
829
        VS_fraction = translate_val(parm_dict['VS_cycle_fraction'],
                                    ['full', '1/2', '1/4', '3/4'],
                                    [1., 0.5, 0.25, 0.75])
Somnath, Suhas's avatar
Somnath, Suhas committed
830
831
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
832
833
834
835
836
837
838
            VS_shift = translate_val(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
        VS_in_out_cond = translate_val(parm_dict['VS_measure_in_field_loops'],
                                       ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
        VS_ACDC_cond = translate_val(parm_dict['VS_mode'],
                                     ['DC modulation mode', 'AC modulation mode with time reversal',
                                      'load user defined VS Wave from file', 'current mode'],
                                     [0, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
839
840
841
842
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
Unknown's avatar
Unknown committed
843
        # FORC_repeats = parm_dict['# of FORC repeats']
Somnath, Suhas's avatar
Somnath, Suhas committed
844
845
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']
Unknown's avatar
Unknown committed
846
847
848

        # % build vector of voltage spectroscopy values

Somnath, Suhas's avatar
Somnath, Suhas committed
849
        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
Unknown's avatar
Unknown committed
850
            VS_amp_vec_1 = np.arange(0, 1 + 1 / (VS_steps / 4), 1 / (VS_steps / 4))
Somnath, Suhas's avatar
Somnath, Suhas committed
851
852
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
Unknown's avatar
Unknown committed
853
854
            VS_amp_vec_4 = VS_amp_vec_1[1:-1] - 1
            vs_amp_vec = VS_amp * (np.hstack((VS_amp_vec_1, VS_amp_vec_2, VS_amp_vec_3, VS_amp_vec_4)))
Unknown's avatar
Unknown committed
855
            # apply phase shift to VS wave
Unknown's avatar
Unknown committed
856
            vs_amp_vec = np.roll(vs_amp_vec, int(np.floor(VS_steps / VS_fraction * VS_shift)))
Unknown's avatar
Unknown committed
857
            # cut VS waveform
Unknown's avatar
Unknown committed
858
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction))]
Unknown's avatar
Unknown committed
859
860
            # repeat VS waveform
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles)
Unknown's avatar
Unknown committed
861
862
            vs_amp_vec = vs_amp_vec + VS_offset

Somnath, Suhas's avatar
Somnath, Suhas committed
863
        elif VS_ACDC_cond == 2:  # AC voltage spectroscopy with time reversal
Unknown's avatar
Unknown committed
864
865
            vs_amp_vec = VS_amp * np.arange(1 / (VS_steps / 2 / VS_fraction), 1 + 1 / (VS_steps / 2 / VS_fraction),
                                            1 / (VS_steps / 2 / VS_fraction))
Somnath, Suhas's avatar
Somnath, Suhas committed
866
            vs_amp_vec = np.roll(vs_amp_vec,
Unknown's avatar
Unknown committed
867
868
                                 int(np.floor(VS_steps / VS_fraction * VS_shift)))  # apply phase shift to VS wave
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction / 2))]  # cut VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
869
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles * 2)  # repeat VS waveform
Unknown's avatar
Unknown committed
870

Somnath, Suhas's avatar
Somnath, Suhas committed
871
        if FORC_cycles > 1:
Unknown's avatar
Unknown committed
872
873
874
875
876
877
878
            vs_amp_vec = vs_amp_vec / np.max(np.abs(vs_amp_vec))
            FORC_cycle_vec = np.arange(0, FORC_cycles + 1, FORC_cycles / (FORC_cycles - 1))
            FORC_A_vec = FORC_cycle_vec * (FORC_A2 - FORC_A1) / FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec * (FORC_B2 - FORC_B1) / FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec - FORC_B_vec) / 2
            FORC_off_vec = (FORC_A_vec + FORC_B_vec) / 2

Somnath, Suhas's avatar
Somnath, Suhas committed
879
880
881
            VS_amp_mat = np.tile(vs_amp_vec, [FORC_cycles, 1])
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(vs_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(vs_amp_vec), 1]).transpose()
Unknown's avatar
Unknown committed
882
883
884
            VS_amp_mat = VS_amp_mat * FORC_amp_mat + FORC_off_mat
            vs_amp_vec = VS_amp_mat.reshape(int(FORC_cycles * VS_cycles * VS_fraction * VS_steps))

Somnath, Suhas's avatar
Somnath, Suhas committed
885
886
        # Build UDVS table:
        if VS_ACDC_cond is 0 or VS_ACDC_cond is 4:  # DC voltage spectroscopy or current mode
Unknown's avatar
Unknown committed
887

Somnath, Suhas's avatar
Somnath, Suhas committed
888
            if VS_ACDC_cond is 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
889
                UD_dc_vec = np.vstack((vs_amp_vec, np.zeros(len(vs_amp_vec))))
Somnath, Suhas's avatar
Somnath, Suhas committed
890
            if VS_ACDC_cond is 4:
Somnath, Suhas's avatar
Somnath, Suhas committed
891
                UD_dc_vec = np.vstack((vs_amp_vec, vs_amp_vec))
Unknown's avatar
Unknown committed
892

Somnath, Suhas's avatar
Somnath, Suhas committed
893
            UD_dc_vec = UD_dc_vec.transpose().reshape(UD_dc_vec.size)
Somnath, Suhas's avatar
Somnath, Suhas committed
894
            num_VS_steps = UD_dc_vec.size
Unknown's avatar
Unknown committed
895

Somnath, Suhas's avatar
Somnath, Suhas committed
896
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'in-field', 'out-of-field']
Somnath, Suhas's avatar
Somnath, Suhas committed
897
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'V', 'V']
Somnath, Suhas's avatar
Somnath, Suhas committed
898
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Unknown's avatar
Unknown committed
899

Somnath, Suhas's avatar
Somnath, Suhas committed
900
901
            udvs_table[:, 0] = np.arange(0, num_VS_steps)  # Python base 0
            udvs_table[:, 1] = UD_dc_vec
Unknown's avatar
Unknown committed
902
903
904
905

            BE_IF_switch = np.abs(np.imag(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))
            BE_OF_switch = np.abs(np.real(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))

Somnath, Suhas's avatar
Somnath, Suhas committed
906
907
908
909
910
911
            if VS_in_out_cond is 0:  # out of field only
                udvs_table[:, 2] = BE_amp * BE_OF_switch
            elif VS_in_out_cond is 1:  # in field only
                udvs_table[:, 2] = BE_amp * BE_IF_switch
            elif VS_in_out_cond is 2:  # both in and out of field
                udvs_table[:, 2] = BE_amp * np.ones(num_VS_steps)
Unknown's avatar
Unknown committed
912

Somnath, Suhas's avatar
Somnath, Suhas committed
913
914
            udvs_table[:, 3] = np.ones(num_VS_steps)  # wave type
            udvs_table[:, 4] = np.ones(num_VS_steps) * BE_signal_type  # wave mod
Unknown's avatar
Unknown committed
915
916
917
918

            udvs_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN') * np.ones(num_VS_steps)

Somnath, Suhas's avatar
Somnath, Suhas committed
919
920
            udvs_table[BE_IF_switch == 1, 5] = udvs_table[BE_IF_switch == 1, 1]
            udvs_table[BE_OF_switch == 1, 6] = udvs_table[BE_IF_switch == 1, 1]
Unknown's avatar
Unknown committed
921

Somnath, Suhas's avatar
Somnath, Suhas committed
922
        elif VS_ACDC_cond is 2:  # AC voltage spectroscopy
Unknown's avatar
Unknown committed
923

Somnath, Suhas's avatar
Somnath, Suhas committed
924
            num_VS_steps = vs_amp_vec.size
Unknown's avatar
Unknown committed
925
926
            half = int(0.5 * num_VS_steps)

Somnath, Suhas's avatar
Somnath, Suhas committed
927
            if num_VS_steps is not half * 2:
928
                raise ValueError('Odd number of UDVS steps found. Exiting!')
Unknown's avatar
Unknown committed
929
930

            UD_dc_vec = VS_offset * np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
931
            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'forward', 'reverse']
Somnath, Suhas's avatar
Somnath, Suhas committed
932
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'A', 'A']
Somnath, Suhas's avatar
Somnath, Suhas committed
933
            udvs_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Unknown's avatar
Unknown committed
934
            udvs_table[:, 0] = np.arange(1, num_VS_steps + 1)
Somnath, Suhas's avatar
Somnath, Suhas committed
935
936
937
            udvs_table[:, 1] = UD_dc_vec
            udvs_table[:, 2] = vs_amp_vec
            udvs_table[:, 3] = np.ones(num_VS_steps)
Unknown's avatar
Unknown committed
938
939
940
941
            udvs_table[:half, 4] = BE_signal_type * np.ones(half)
            udvs_table[half:, 4] = -1 * BE_signal_type * np.ones(half)
            udvs_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            udvs_table[:, 6] = float('NaN') * np.ones(num_VS_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
942
943
            udvs_table[:half, 5] = vs_amp_vec[:half]
            udvs_table[half:, 6] = vs_amp_vec[half:]
Unknown's avatar
Unknown committed
944

Somnath, Suhas's avatar
Somnath, Suhas committed
945
946
        return UD_VS_table_label, UD_VS_table_unit, udvs_table

947