be_odf_relaxation.py 28.9 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Thursday May 26 11:23:00 2016

@author:  Rama Vasudevan, Suhas Somnath
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

10
from os import path, remove  # File Path formatting
11
12
from warnings import warn

Unknown's avatar
Unknown committed
13
import numpy as np  # For array operations
14
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15

16
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, generatePlotGroups, createSpecVals, nf32
17
from .translator import Translator  # Because this class extends the abstract Translator class
18
from .utils import make_position_mat, get_position_slicing, generate_dummy_main_parms
Chris Smith's avatar
Chris Smith committed
19
from ..be_hdf_utils import maxReadPixels
Somnath, Suhas's avatar
Somnath, Suhas committed
20
from ..hdf_utils import getH5DsetRefs
21
from ..io_hdf5 import ioHDF5  # Now the translator is responsible for writing the data.
Unknown's avatar
Unknown committed
22
23
# The building blocks for defining hierarchical storage in the H5 file
from ..microdata import MicroDataGroup, MicroDataset
Somnath, Suhas's avatar
Somnath, Suhas committed
24

Chris Smith's avatar
Chris Smith committed
25

Somnath, Suhas's avatar
Somnath, Suhas committed
26
27
28
29
30
31
32
33
class BEodfRelaxationTranslator(Translator):
    """
    Translates old Relaxation data into the new H5 format. This is for the files generated from
    the old BEPSDAQ program utilizing two cards simultaneously.
    At present, this version of the translator only works for Out of field measurements
    It will not work for in-field. This should be fixed at a later date.
    
    """
Unknown's avatar
Unknown committed
34

Somnath, Suhas's avatar
Somnath, Suhas committed
35
36
37
38
39
40
41
42
43
44
45
46
    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False):
        """
        Basic method that translates .dat data file(s) to a single .h5 file
        
        Inputs:
            file_path -- Absolute file path for one of the data files. 
            It is assumed that this file is of the OLD data format. 
            
        Outputs:
            Nothing
        """
        (folder_path, basename) = path.split(file_path)
47
        (basename, path_dict) = self._parse_file_path(file_path)
Unknown's avatar
Unknown committed
48
49

        h5_path = path.join(folder_path, basename + '.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
50
51
52

        isBEPS = True
        parm_dict = self.__getParmsFromOldMat(path_dict['old_mat_parms'])
Unknown's avatar
Unknown committed
53
54
55
56
57

        ignored_plt_grps = ['in-field']  # Here we assume that there is no in-field.
        # If in-field data is captured then the translator would have to be modified.

        # Technically, we could do away with this if statement, as isBEPS is always true for this translation
Somnath, Suhas's avatar
Somnath, Suhas committed
58
59
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'
Unknown's avatar
Unknown committed
60

Somnath, Suhas's avatar
Somnath, Suhas committed
61
            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'
Unknown's avatar
Unknown committed
62

Somnath, Suhas's avatar
Somnath, Suhas committed
63
64
65
            if not std_expt:
                warn('This translator does not handle user defined voltage spectroscopy')
                return
Unknown's avatar
Unknown committed
66
67
68
69

            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode'])

            # Check file sizes:
Somnath, Suhas's avatar
Somnath, Suhas committed
70
        if 'read_real' in path_dict.keys():
Unknown's avatar
Unknown committed
71
72
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
Somnath, Suhas's avatar
Somnath, Suhas committed
73
74
75
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])
Unknown's avatar
Unknown committed
76

Somnath, Suhas's avatar
Somnath, Suhas committed
77
78
79
        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

Unknown's avatar
Unknown committed
80
81
82
83
84
85
86
87
88
89
        add_pix = False
        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
        num_pix = num_rows * num_cols
        tot_bins = real_size / (num_pix * 4)  # Finding bins by simple division of entire datasize

        # Check for case where only a single pixel is missing.
        check_bins = real_size / ((num_pix - 1) * 4)

        if tot_bins % 1 and check_bins % 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
90
            warn('Aborting! Some parameter appears to have changed in-between')
Unknown's avatar
Unknown committed
91
            return
Somnath, Suhas's avatar
Somnath, Suhas committed
92
        elif not tot_bins % 1:
Unknown's avatar
Unknown committed
93
            #             Everything's ok
Somnath, Suhas's avatar
Somnath, Suhas committed
94
95
96
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
Unknown's avatar
Unknown committed
97
98
99
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.')
            add_pix = True

Somnath, Suhas's avatar
Somnath, Suhas committed
100
        tot_bins = int(tot_bins)
Unknown's avatar
Unknown committed
101
        (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__readOldMatBEvecs(path_dict['old_mat_parms'])
Unknown's avatar
Unknown committed
102
103
104
105
        """
        Because this is the old data format and there is a discrepancy in the number of bins (they seem to be 2 less 
        than the actual number), we need to re-calculate it based on the available data. This is done below.
        """
Unknown's avatar
Unknown committed
106
107

        band_width = parm_dict['BE_band_width_[Hz]'] * (0.5 - parm_dict['BE_band_edge_trim'])
Somnath, Suhas's avatar
Somnath, Suhas committed
108
        st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
Unknown's avatar
Unknown committed
109
        en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width
Somnath, Suhas's avatar
Somnath, Suhas committed
110
        bin_freqs = np.linspace(st_f, en_f, len(bin_inds), dtype=np.float32)
Unknown's avatar
Unknown committed
111

Somnath, Suhas's avatar
Somnath, Suhas committed
112
113
114
115
116
        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)
Unknown's avatar
Unknown committed
117

Somnath, Suhas's avatar
Somnath, Suhas committed
118
        self.FFT_BE_wave = bin_FFT
119
120
        pos_mat = make_position_mat([num_cols, num_rows])
        pos_slices = get_position_slicing(['X', 'Y'], num_pix)
Unknown's avatar
Unknown committed
121
122

        ds_ex_wfm = MicroDataset('Excitation_Waveform', ex_wfm)
Somnath, Suhas's avatar
Somnath, Suhas committed
123
124
125
126
127
128
        ds_pos_ind = MicroDataset('Position_Indices', pos_mat, dtype=np.uint32)
        ds_pos_ind.attrs['labels'] = pos_slices
        ds_pos_val = MicroDataset('Position_Values', np.float32(pos_mat))
        ds_pos_val.attrs['labels'] = pos_slices

        (UDVS_labs, UDVS_units, UDVS_mat) = self.__buildUDVSTable(parm_dict)
Unknown's avatar
Unknown committed
129

Unknown's avatar
Unknown committed
130
        # Remove the unused plot group columns before proceeding:
Somnath, Suhas's avatar
Somnath, Suhas committed
131
        (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
Unknown's avatar
Unknown committed
132
133
134

        spec_inds = np.zeros(shape=(2, tot_bins), dtype=np.uint)

Unknown's avatar
Unknown committed
135
136
        # Will assume that all excitation waveforms have same number of bins
        # Here, the denominator is 2 because only out of field measruements. For IF + OF, should be 1
Unknown's avatar
Unknown committed
137
138
139
140
        num_actual_udvs_steps = UDVS_mat.shape[0] / 2
        bins_per_step = tot_bins / num_actual_udvs_steps

        # Some more checks
Somnath, Suhas's avatar
Somnath, Suhas committed
141
142
143
144
145
        if bins_per_step % 1:
            warn('Non integer number of bins per step!')
            return
        else:
            bins_per_step = int(bins_per_step)
Unknown's avatar
Unknown committed
146

Somnath, Suhas's avatar
Somnath, Suhas committed
147
        num_actual_udvs_steps = int(num_actual_udvs_steps)
Unknown's avatar
Unknown committed
148
149
150
151
152
153
154
155

        stind = 0
        for step_index in xrange(UDVS_mat.shape[0]):
            if UDVS_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                continue  # skip
            spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=np.uint32)  # Bin step
            spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step,
                                                                             dtype=np.uint32)  # UDVS step
Somnath, Suhas's avatar
Somnath, Suhas committed
156
            stind += bins_per_step
Unknown's avatar
Unknown committed
157
158
        del stind, step_index

Somnath, Suhas's avatar
Somnath, Suhas committed
159
160
161
        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
162
        parm_dict['num_udvs_steps'] = num_actual_udvs_steps
Unknown's avatar
Unknown committed
163
164

        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
165
        for col_ind, col_name in enumerate(UDVS_labs):
Unknown's avatar
Unknown committed
166
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind + 1))
Somnath, Suhas's avatar
Somnath, Suhas committed
167
168
169
        ds_UDVS = MicroDataset('UDVS', UDVS_mat)
        ds_UDVS.attrs['labels'] = udvs_slices
        ds_UDVS.attrs['units'] = UDVS_units
Unknown's avatar
Unknown committed
170

Somnath, Suhas's avatar
Somnath, Suhas committed
171
        ds_spec_mat = MicroDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
Unknown's avatar
Unknown committed
172
173
174
        ds_spec_mat.attrs['labels'] = {'UDVS_Step': (slice(1, 2), slice(None)), 'Bin': (slice(0, 1), slice(None))}
        ds_bin_steps = MicroDataset('Bin_Step', np.arange(bins_per_step, dtype=np.uint32), dtype=np.uint32)

Somnath, Suhas's avatar
Somnath, Suhas committed
175
        # Need to add the Bin Waveform type - infer from UDVS        
Unknown's avatar
Unknown committed
176
        exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
Somnath, Suhas's avatar
Somnath, Suhas committed
177

Unknown's avatar
Unknown committed
178
179
        ds_bin_inds = MicroDataset('Bin_Indices', bin_inds, dtype=np.uint32)
        ds_bin_freq = MicroDataset('Bin_Frequencies', bin_freqs)
Somnath, Suhas's avatar
Somnath, Suhas committed
180
181
        ds_bin_FFT = MicroDataset('Bin_FFT', bin_FFT)
        ds_wfm_typ = MicroDataset('Bin_Wfm_Type', exec_bin_vec)
Unknown's avatar
Unknown committed
182

Somnath, Suhas's avatar
Somnath, Suhas committed
183
        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
Unknown's avatar
Unknown committed
184
185
186
187
188
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_names = createSpecVals(UDVS_mat, spec_inds,
                                                                                                bin_freqs,
                                                                                                exec_bin_vec,
                                                                                                parm_dict, UDVS_labs,
                                                                                                UDVS_units)
Unknown's avatar
Unknown committed
189

Somnath, Suhas's avatar
Somnath, Suhas committed
190
191
        spec_vals_slices = dict()
        for row_ind, row_name in enumerate(spec_vals_labs):
Unknown's avatar
Unknown committed
192
193
            spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
        ds_spec_vals_mat = MicroDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
Somnath, Suhas's avatar
Somnath, Suhas committed
194
195
        ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
        ds_spec_vals_mat.attrs['units'] = spec_vals_units
Unknown's avatar
Unknown committed
196

Somnath, Suhas's avatar
Somnath, Suhas committed
197
        # Noise floor should be of shape: (udvs_steps x 3 x positions)
Chris Smith's avatar
Chris Smith committed
198
199
        ds_noise_floor = MicroDataset('Noise_Floor', np.zeros(shape=(num_pix, num_actual_udvs_steps), dtype=nf32),
                                      chunking=(1, num_actual_udvs_steps))
Unknown's avatar
Unknown committed
200

Somnath, Suhas's avatar
Somnath, Suhas committed
201
202
203
204
205
206
207
208
209
        """ 
        ONLY ALLOCATING SPACE FOR MAIN DATA HERE!
        Chunk by each UDVS step - this makes it easy / quick to:
            1. read data for a single UDVS step from all pixels
            2. read an entire / multiple pixels at a time
        The only problem is that a typical UDVS step containing 50 steps occupies only 400 bytes.
        This is smaller than the recommended chunk sizes of 10,000 - 999,999 bytes
        meaning that the metadata would be very substantial.
        This assumption is fine since we almost do not handle any user defined cases
Unknown's avatar
Unknown committed
210
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
211
212
213
214
215
216
217
218

        """
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares of UDVS steps by pixels.
        N is determined dinamically based on the dimensions of the dataset.  Currently it is set such that individual
        chunks are less than 10kB in size.
        
        Chris Smith -- csmith55@utk.edu
        """
Unknown's avatar
Unknown committed
219
        pixel_chunking = maxReadPixels(10240, num_pix * num_actual_udvs_steps,
Unknown's avatar
Unknown committed
220
                                       bins_per_step, np.dtype('complex64').itemsize)
Somnath, Suhas's avatar
Somnath, Suhas committed
221
222
223
        chunking = np.floor(np.sqrt(pixel_chunking))
        chunking = max(1, chunking)
        chunking = min(num_actual_udvs_steps, num_pix, chunking)
Unknown's avatar
Unknown committed
224
225
226
        ds_main_data = MicroDataset('Raw_Data', data=[], maxshape=(num_pix, tot_bins), dtype=np.complex64,
                                    chunking=(chunking, chunking * bins_per_step), compression='gzip')

Somnath, Suhas's avatar
Somnath, Suhas committed
227
        chan_grp = MicroDataGroup('Channel_')
228
        chan_grp.attrs['Channel_Input'] = parm_dict['IO_Analog_Input_1']
Somnath, Suhas's avatar
Somnath, Suhas committed
229
230
        chan_grp.addChildren([ds_main_data, ds_noise_floor])
        chan_grp.addChildren([ds_ex_wfm, ds_pos_ind, ds_pos_val, ds_spec_mat, ds_UDVS,
Unknown's avatar
Unknown committed
231
232
                              ds_bin_steps, ds_bin_inds, ds_bin_freq, ds_bin_FFT, ds_wfm_typ, ds_spec_vals_mat])

Somnath, Suhas's avatar
Somnath, Suhas committed
233
234
235
236
        # technically should change the date, etc.
        meas_grp = MicroDataGroup('Measurement_')
        meas_grp.attrs = parm_dict
        meas_grp.addChildren([chan_grp])
Unknown's avatar
Unknown committed
237

Somnath, Suhas's avatar
Somnath, Suhas committed
238
        spm_data = MicroDataGroup('')
239
        global_parms = generate_dummy_main_parms()
Unknown's avatar
Unknown committed
240
241
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
242
        global_parms['experiment_date'] = parm_dict['File_date_and_time']
Unknown's avatar
Unknown committed
243
244
245
246
247

        # assuming that the experiment was completed:
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
        global_parms['data_type'] = parm_dict['data_type']  # self.__class__.__name__
Somnath, Suhas's avatar
Somnath, Suhas committed
248
        global_parms['translator'] = 'ODF'
Unknown's avatar
Unknown committed
249

Somnath, Suhas's avatar
Somnath, Suhas committed
250
251
252
253
254
        spm_data.attrs = global_parms
        spm_data.addChildren([meas_grp])

        if path.exists(h5_path):
            remove(h5_path)
Unknown's avatar
Unknown committed
255

Somnath, Suhas's avatar
Somnath, Suhas committed
256
257
        # Write everything except for the main data.
        self.hdf = ioHDF5(h5_path)
Unknown's avatar
Unknown committed
258
259
        # self.hdf.clear() #Doesn't seem to work

Somnath, Suhas's avatar
Somnath, Suhas committed
260
        h5_refs = self.hdf.writeData(spm_data)
Unknown's avatar
Unknown committed
261

Somnath, Suhas's avatar
Somnath, Suhas committed
262
        self.ds_main = getH5DsetRefs(['Raw_Data'], h5_refs)[0]
Unknown's avatar
Unknown committed
263
264
265
266
267

        # Now doing linkrefs:
        aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values',
                        'Spectroscopic_Indices', 'UDVS', 'Bin_Step', 'Bin_Indices',
                        'Bin_Frequencies', 'Bin_FFT', 'Bin_Wfm_Type', 'Noise_Floor', 'Spectroscopic_Values']
Somnath, Suhas's avatar
Somnath, Suhas committed
268
        self.hdf.linkRefs(self.ds_main, getH5DsetRefs(aux_ds_names, h5_refs))
Unknown's avatar
Unknown committed
269

Somnath, Suhas's avatar
Somnath, Suhas committed
270
271
272
        self.mean_resp = np.zeros(shape=(self.ds_main.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.ds_main.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.ds_main.shape[0]), dtype=np.float32)
Unknown's avatar
Unknown committed
273

Somnath, Suhas's avatar
Somnath, Suhas committed
274
        # Now read the raw data files:
275
        self._read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict)
Somnath, Suhas's avatar
Somnath, Suhas committed
276
        self.hdf.flush()
Unknown's avatar
Unknown committed
277

Unknown's avatar
Unknown committed
278
279
280
        generatePlotGroups(self.ds_main, self.hdf, self.mean_resp, folder_path, basename, self.max_resp,
                           self.min_resp, max_mem_mb=self.max_ram, spec_label=spec_label, show_plots=show_plots,
                           save_plots=save_plots, do_histogram=do_histogram,
Unknown's avatar
Unknown committed
281
282
                           ignore_plot_groups=ignored_plt_grps)  # We ignored in-field plot group.

Somnath, Suhas's avatar
Somnath, Suhas committed
283
        self.hdf.close()
Unknown's avatar
Unknown committed
284

Somnath, Suhas's avatar
Somnath, Suhas committed
285
        return h5_path
Unknown's avatar
Unknown committed
286

287
    def _read_data(self, real_path, imag_path, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
288
289
290
291
292
293
294
295
296
297
        """
        Reads the imaginary and real data files one pixel at a time andwrites to the H5 dataset.
        
        Inputs:
            real_path -- file path of the .dat file containing the real component of the data
            imag_path -- file path of the .dat file containing the imaginary component of the data
            parm_dict--dictionary of parameters for the experiment            
            
        Outputs: None
        """
Unknown's avatar
Unknown committed
298
299
300
301
302
303
304
        print('---- reading data one pixel at a time----------')

        num_pix = int(parm_dict['grid_num_rows']) * int(parm_dict['grid_num_cols'])
        # print 'Number of rows is: ', parm_dict['grid_num_rows']
        # print 'Number of cols is: ', parm_dict['grid_num_cols']
        # print 'Rows * cols is:', int(parm_dict['grid_num_rows'])*int(parm_dict['grid_num_cols'])

Somnath, Suhas's avatar
Somnath, Suhas committed
305
306
        if path.getsize(real_path) != path.getsize(imag_path):
            print('Sizes of real and imaginary files NOT matching!!!!')
Unknown's avatar
Unknown committed
307
        if 1.0 * path.getsize(real_path) % num_pix != 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
308
            print('Incomplete dataset!!!')
Unknown's avatar
Unknown committed
309
310

        bytes_per_pix = path.getsize(real_path) / num_pix
Somnath, Suhas's avatar
Somnath, Suhas committed
311
312
313
        f_real = open(real_path, "rb")
        f_imag = open(imag_path, "rb")

Unknown's avatar
Unknown committed
314
315
        for pix_ind in range(num_pix):
            print('Reading pixel #{}, file position {}'.format(pix_ind, hex(pix_ind * bytes_per_pix)))
Unknown's avatar
Unknown committed
316
            pix_vec = np.fromstring(f_real.read(int(bytes_per_pix)), dtype='f') + \
Unknown's avatar
Unknown committed
317
                1j * np.fromstring(f_imag.read(int(bytes_per_pix)), dtype='f')
Unknown's avatar
Unknown committed
318

Somnath, Suhas's avatar
Somnath, Suhas committed
319
            # Make chronologically correct
Unknown's avatar
Unknown committed
320
321
            pix_mat = np.reshape(pix_vec, (parm_dict['BE_bins_per_read'],
                                           parm_dict['VS_steps_per_full_cycle'], parm_dict['BE_repeats']))
Unknown's avatar
Unknown committed
322
            pix_mat_temp = np.transpose(pix_mat, (1, 2, 0))
Somnath, Suhas's avatar
Somnath, Suhas committed
323
            pix_vec2 = np.reshape(pix_mat_temp, -1)
Unknown's avatar
Unknown committed
324
325

            # Calculate the mean, min, max
Somnath, Suhas's avatar
Somnath, Suhas committed
326
327
            self.max_resp[pix_ind] = np.max(np.abs(pix_vec2))
            self.min_resp[pix_ind] = np.min(np.abs(pix_vec2))
Unknown's avatar
Unknown committed
328
            self.mean_resp = (1 / (pix_ind + 1)) * (pix_vec2 + pix_ind * self.mean_resp)
Somnath, Suhas's avatar
Somnath, Suhas committed
329
330

            # Write to file now
Unknown's avatar
Unknown committed
331
            self.ds_main[pix_ind, :] = np.complex64(pix_vec2)
Somnath, Suhas's avatar
Somnath, Suhas committed
332
            self.hdf.flush()
Unknown's avatar
Unknown committed
333

Somnath, Suhas's avatar
Somnath, Suhas committed
334
335
        f_real.close()
        f_imag.close()
Unknown's avatar
Unknown committed
336

Somnath, Suhas's avatar
Somnath, Suhas committed
337
338
        print('Finished writing data to .h5')

Unknown's avatar
Unknown committed
339
340
    @staticmethod
    def __readOldMatBEvecs(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
        """
    Returns information about the excitation BE waveform present in the .mat file
    
    Inputs:
        filepath -- Absolute filepath of the .mat parameter file
    
    Outputs:
        Tuple -- (bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full)\n
        bin_inds -- Bin indices\n
        bin_w -- Excitation bin Frequencies\n
        bin_FFT -- FFT of the BE waveform for the excited bins\n
        BE_wave -- Band Excitation waveform\n
        dc_amp_vec_full -- spectroscopic waveform. 
        This information will be necessary for fixing the UDVS for AC modulation for example
        """
Unknown's avatar
Unknown committed
356
357

        matread = loadmat(file_path, squeeze_me=True)
Somnath, Suhas's avatar
Somnath, Suhas committed
358
        BE_wave = matread['BE_wave_1']
Unknown's avatar
Unknown committed
359
        bin_inds = matread['bin_ind_s'] - 1  # Python base 0. note also _s, for this case
Somnath, Suhas's avatar
Somnath, Suhas committed
360
361
362
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
Unknown's avatar
Unknown committed
363
364
        bin_FFT = np.conjugate(FFT_full[bin_inds])

Unknown's avatar
Unknown committed
365
        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full
Unknown's avatar
Unknown committed
366

367
    def _parse_file_path(self, data_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
368
369
370
371
372
373
374
375
376
377
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary
        
        Inputs:
            data_filepath: Absolute path of the real / imaginary data file (.dat)
        Outputs:
            Tuple (basename, path_dict)
        """
        (folder_path, basename) = path.split(data_filepath)
Unknown's avatar
Unknown committed
378
        (super_folder, basename) = path.split(folder_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
379
380
381
382

        if basename.endswith('_c'):
            # Old old data format where the folder ended with a _d for some reason
            base_name = basename[:-2]
Unknown's avatar
Unknown committed
383

Somnath, Suhas's avatar
Somnath, Suhas committed
384
385
386
387
388
389
390
        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()
Unknown's avatar
Unknown committed
391

Somnath, Suhas's avatar
Somnath, Suhas committed
392
393
        real_path = path.join(folder_path, base_name + '_sub_real.dat')
        imag_path = path.join(folder_path, base_name + '_sub_imag.dat')
Unknown's avatar
Unknown committed
394

Somnath, Suhas's avatar
Somnath, Suhas committed
395
396
        path_dict['read_real'] = real_path
        path_dict['read_imag'] = imag_path
Unknown's avatar
Unknown committed
397
398
        path_dict['old_mat_parms'] = data_filepath

Unknown's avatar
Unknown committed
399
        return basename, path_dict
Unknown's avatar
Unknown committed
400

Unknown's avatar
Unknown committed
401
402
    @staticmethod
    def __getParmsFromOldMat(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
403
404
405
406
407
408
409
410
411
412
413
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it
        
        Inputs:
            file_path -- absolute filepath of the .mat file containing the parameters
            
        Outputs -- dictionary containing parameters
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)
Unknown's avatar
Unknown committed
414
415
416

        parm_dict['IO_rate'] = str(int(matread['AO_rate'] / 1E+6)) + ' MHz'

Somnath, Suhas's avatar
Somnath, Suhas committed
417
418
419
420
421
        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = int(position_vec[2])
        parm_dict['grid_num_cols'] = int(position_vec[3])
Unknown's avatar
Unknown committed
422

Somnath, Suhas's avatar
Somnath, Suhas committed
423
424
        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
Unknown's avatar
Unknown committed
425
            parm_dict['grid_num_rows'] = int(position_vec[0])  # set to number of present cols and rows
Somnath, Suhas's avatar
Somnath, Suhas committed
426
            parm_dict['grid_num_cols'] = int(position_vec[1])
Unknown's avatar
Unknown committed
427

Somnath, Suhas's avatar
Somnath, Suhas committed
428
429
430
431
432
433
434
435
        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
Unknown's avatar
Unknown committed
436
437
438
439
440
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
        parm_dict['BE_band_edge_trim'] = -1 * BE_parm_vec_1[6]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
        parm_dict['BE_repeats'] = 2 ** int(BE_parm_vec_1[8])
        parm_dict['File_date_and_time'] = 0  # For now ignoring.
Somnath, Suhas's avatar
Somnath, Suhas committed
441
442
        parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        assembly_parm_vec = matread['assembly_parm_vec']
Unknown's avatar
Unknown committed
443

Somnath, Suhas's avatar
Somnath, Suhas committed
444
445
446
447
448
449
        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
Unknown's avatar
Unknown committed
450

Somnath, Suhas's avatar
Somnath, Suhas committed
451
452
453
454
455
        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'
Unknown's avatar
Unknown committed
456
457
458
459

        # num_driving_bands = assembly_parm_vec[0] # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1] # 0 parallel 1 series

Somnath, Suhas's avatar
Somnath, Suhas committed
460
461
        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']
Unknown's avatar
Unknown committed
462
463
464
465
466
467
468
469
470

        VS_start_V = VS_parms[4]
        VS_start_loop_amp = VS_parms[5]
        VS_final_loop_amp = VS_parms[6]
        # VS_read_write_ratio = VS_parms[8] #1 <- SS_read_write_ratio

        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3]
        parm_dict['VS_steps_per_full_cycle'] = len(dc_amp_vec_full)  # VS_parms[7]
Somnath, Suhas's avatar
Somnath, Suhas committed
471
        parm_dict['VS_cycle_fraction'] = 'full'
Unknown's avatar
Unknown committed
472
        parm_dict['VS_cycle_phase_shift'] = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
473
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
Unknown's avatar
Unknown committed
474
475
476
477
478
479
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0

Somnath, Suhas's avatar
Somnath, Suhas committed
480
481
        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
Unknown's avatar
Unknown committed
482
483
484
            parm_dict['VS_amplitude_[V]'] = 0.5 * (max(dc_amp_vec_full) -
                                                   min(dc_amp_vec_full))  # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)
Somnath, Suhas's avatar
Somnath, Suhas committed
485
486
487
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
Unknown's avatar
Unknown committed
488
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
489
            parm_dict['VS_offset_[V]'] = 0
Unknown's avatar
Unknown committed
490
491
492
493
494
495
            parm_dict['VS_number_of_cycles'] = 1
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
496
497
498
        elif VS_parms[0] == 2:
            # AC mode 
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
Unknown's avatar
Unknown committed
499
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
Unknown's avatar
Unknown committed
500
501
            parm_dict[
                'VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
Somnath, Suhas's avatar
Somnath, Suhas committed
502
503
        else:
            parm_dict['VS_mode'] = 'Custom'
Unknown's avatar
Unknown committed
504

Somnath, Suhas's avatar
Somnath, Suhas committed
505
        return parm_dict
Unknown's avatar
Unknown committed
506
507

    def __buildUDVSTable(self, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
508
509
510
511
512
513
514
515
516
517
518
        """
        Generates the UDVS table using the parameters
        
        Inputs:
            parm_dict -- Dictionary of parameters present in the text files
        
        Outputs:
            tuple (labels, table)
            labels -- List of strings - Labels for columns in the UDVS table
            table -- UDVS data table
        """
Unknown's avatar
Unknown committed
519

Somnath, Suhas's avatar
Somnath, Suhas committed
520
521
522
523
        def translateVal(target, strvals, numvals):
            """
            Internal function - Interprets the provided value using the provided lookup table
            """
Unknown's avatar
Unknown committed
524

Somnath, Suhas's avatar
Somnath, Suhas committed
525
            if len(strvals) is not len(numvals):
Unknown's avatar
Unknown committed
526
527
                return None
            for strval, fltval in zip(strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
528
529
                if target == strval:
                    return fltval
Unknown's avatar
Unknown committed
530
531
            return None  # not found in list

Unknown's avatar
Unknown committed
532
        # Extract values from parm text file
Unknown's avatar
Unknown committed
533
        BE_signal_type = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
534
535
536
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type
Unknown's avatar
Unknown committed
537

Somnath, Suhas's avatar
Somnath, Suhas committed
538
        BE_amp = parm_dict['BE_amplitude_[V]']
Unknown's avatar
Unknown committed
539

Somnath, Suhas's avatar
Somnath, Suhas committed
540
541
542
543
        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
Unknown's avatar
Unknown committed
544
        VS_fraction = translateVal(parm_dict['VS_cycle_fraction'], ['full', '1/2', '1/4', '3/4'], [1., 0.5, 0.25, 0.75])
Somnath, Suhas's avatar
Somnath, Suhas committed
545
546
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
Unknown's avatar
Unknown committed
547
            VS_shift = translateVal(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
Unknown's avatar
Unknown committed
548
        VS_in_out_cond = translateVal(parm_dict['VS_measure_in_field_loops'],
Unknown's avatar
Unknown committed
549
                                      ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
Unknown's avatar
Unknown committed
550
        VS_ACDC_cond = translateVal(parm_dict['VS_mode'],
Unknown's avatar
Unknown committed
551
552
553
                                    ['DC modulation mode', 'AC modulation mode with time reversal',
                                     'load user defined VS Wave from file', 'current mode'],
                                    [0, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
554
555
556
557
558
559
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']
Unknown's avatar
Unknown committed
560
561
562
563
564

        # % build vector of voltage spectroscopy values

        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
            VS_amp_vec_1 = np.arange(0, 1 + 1 / (VS_steps / 4), 1 / (VS_steps / 4))
Somnath, Suhas's avatar
Somnath, Suhas committed
565
566
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
Unknown's avatar
Unknown committed
567
568
569
570
571
572
573
574
            VS_amp_vec_4 = VS_amp_vec_1[1:-1] - 1
            VS_amp_vec = VS_amp * (np.hstack((VS_amp_vec_1, VS_amp_vec_2, VS_amp_vec_3, VS_amp_vec_4)))
            VS_amp_vec = np.roll(VS_amp_vec,
                                 int(np.floor(VS_steps / VS_fraction * VS_shift)))  # apply phase shift to VS wave
            VS_amp_vec = VS_amp_vec[:int(np.floor(VS_steps * VS_fraction))]  # cut VS waveform
            VS_amp_vec = np.tile(VS_amp_vec, VS_cycles)  # repeat VS waveform
            VS_amp_vec = VS_amp_vec + VS_offset

Somnath, Suhas's avatar
Somnath, Suhas committed
575
        if FORC_cycles > 1:
Unknown's avatar
Unknown committed
576
577
578
579
580
581
582
583
584
585
586
587
588
            VS_amp_vec = VS_amp_vec / np.max(np.abs(VS_amp_vec))
            FORC_cycle_vec = np.arange(0, FORC_cycles + 1, FORC_cycles / (FORC_cycles - 1))
            FORC_A_vec = FORC_cycle_vec * (FORC_A2 - FORC_A1) / FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec * (FORC_B2 - FORC_B1) / FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec - FORC_B_vec) / 2
            FORC_off_vec = (FORC_A_vec + FORC_B_vec) / 2

            VS_amp_mat = np.tile(VS_amp_vec, [FORC_cycles, 1])
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(VS_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(VS_amp_vec), 1]).transpose()
            VS_amp_mat = VS_amp_mat * FORC_amp_mat + FORC_off_mat
            VS_amp_vec = VS_amp_mat.reshape(int(FORC_cycles * VS_cycles * VS_fraction * VS_steps))

Somnath, Suhas's avatar
Somnath, Suhas committed
589
        BE_repeats = parm_dict['BE_repeats']
Unknown's avatar
Unknown committed
590
591
592
593
594
595
596
597
598
        total_steps = len(VS_amp_vec) * BE_repeats  # Needed for relaxation datasets

        # % Build UDVS table:
        if VS_ACDC_cond is 0 or VS_ACDC_cond is 4:  # relaxation measurements

            num_VS_steps = total_steps * 2  # To account for IF and OOF

            UD_VS_table_label = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'in-field', 'out-of-field']
            UD_VS_table = np.zeros(shape=(num_VS_steps, 7), dtype=np.float32)
Somnath, Suhas's avatar
Somnath, Suhas committed
599
            UD_VS_table_unit = ['', 'V', 'A', '', '', 'V', 'V']
Unknown's avatar
Unknown committed
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625

            UD_VS_table[:, 0] = np.arange(0, num_VS_steps)  # Python base 0
            for step_num in np.arange(0, VS_steps):
                step_values = (np.arange(int(step_num) * int(BE_repeats) * 2,
                                         (int(step_num) + 1) * int(BE_repeats) * 2))
                UD_VS_table[step_values, 1] = VS_amp_vec[step_num]

            BE_IF_switch = np.abs(np.imag(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))
            BE_OF_switch = np.abs(np.real(np.exp(1j * np.pi / 2 * np.arange(1, num_VS_steps + 1))))

            if VS_in_out_cond is 0:  # out of field only
                UD_VS_table[:, 2] = BE_amp * BE_OF_switch
            elif VS_in_out_cond is 1:  # in field only
                UD_VS_table[:, 2] = BE_amp * BE_IF_switch
            elif VS_in_out_cond is 2:  # both in and out of field
                UD_VS_table[:, 2] = BE_amp * np.ones(num_VS_steps)

            UD_VS_table[:, 3] = np.ones(num_VS_steps)  # wave type
            UD_VS_table[:, 4] = np.ones(num_VS_steps) * BE_signal_type  # wave mod

            UD_VS_table[:, 5] = float('NaN') * np.ones(num_VS_steps)
            UD_VS_table[:, 6] = float('NaN') * np.ones(num_VS_steps)

            UD_VS_table[BE_IF_switch == 1, 5] = UD_VS_table[BE_IF_switch == 1, 1]
            UD_VS_table[BE_OF_switch == 1, 6] = UD_VS_table[BE_IF_switch == 1, 1]

Unknown's avatar
Unknown committed
626
        return UD_VS_table_label, UD_VS_table_unit, UD_VS_table