be_odf.py 52.5 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:24:12 2015

@author: Suhas Somnath, Stephen Jesse
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

Somnath, Suhas's avatar
Somnath, Suhas committed
10
from os import path, listdir, remove
11
from warnings import warn
12
import h5py
Somnath, Suhas's avatar
Somnath, Suhas committed
13
14
import numpy as np
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15

16
from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
17
    createSpecVals, requires_conjugate, nf32
18
19
20
from pyUSID.io.translator import Translator, generate_dummy_main_parms
from pyUSID.io.write_utils import INDICES_DTYPE, VALUES_DTYPE, Dimension, calc_chunks
from pyUSID.io.hdf_utils import write_ind_val_dsets, write_main_dataset, write_region_references, \
21
22
23
24
    create_indexed_group, write_simple_attrs, write_book_keeping_attrs, copy_attributes,\
    write_reduced_spec_dsets
from pyUSID.io.usi_data import USIDataset
from pyUSID.io.io_utils import get_available_memory
25

Somnath, Suhas's avatar
Somnath, Suhas committed
26
27
28
29
30
class BEodfTranslator(Translator):
    """
    Translates either the Band Excitation (BE) scan or Band Excitation 
    Polarization Switching (BEPS) data format from the old data format(s) to .h5
    """
Unknown's avatar
Unknown committed
31

Chris Smith's avatar
Chris Smith committed
32
33
34
    def __init__(self, *args, **kwargs):
        super(BEodfTranslator, self).__init__(*args, **kwargs)
        self.h5_raw = None
35
        self.num_rand_spectra = kwargs.pop('num_rand_spectra', 1000)
Unknown's avatar
Unknown committed
36
37
38
        self.FFT_BE_wave = None
        self.signal_type = None
        self.expt_type = None
Chris Smith's avatar
Chris Smith committed
39

40
    def translate(self, file_path, show_plots=True, save_plots=True, do_histogram=False, verbose=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
41
42
43
44
45
46
47
48
49
50
51
52
53
54
        """
        Translates .dat data file(s) to a single .h5 file
        
        Parameters
        -------------
        file_path : String / Unicode
            Absolute file path for one of the data files. 
            It is assumed that this file is of the OLD data format.
        show_plots : (optional) Boolean
            Whether or not to show intermediate plots
        save_plots : (optional) Boolean
            Whether or not to save plots to disk
        do_histogram : (optional) Boolean
            Whether or not to construct histograms to visualize data quality. Note - this takes a fair amount of time
55
56
        verbose : (optional) Boolean
            Whether or not to print statements
Somnath, Suhas's avatar
Somnath, Suhas committed
57
58
59
60
61
62
            
        Returns
        ----------
        h5_path : String / Unicode
            Absolute path of the resultant .h5 file
        """
63
        file_path = path.abspath(file_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
64
        (folder_path, basename) = path.split(file_path)
65
        (basename, path_dict) = self._parse_file_path(file_path)
Unknown's avatar
Unknown committed
66

Somnath, Suhas's avatar
Somnath, Suhas committed
67
        h5_path = path.join(folder_path, basename + '.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
68
69
        tot_bins_multiplier = 1
        udvs_denom = 2
Unknown's avatar
Unknown committed
70

Somnath, Suhas's avatar
Somnath, Suhas committed
71
        if 'parm_txt' in path_dict.keys():
Unknown's avatar
Unknown committed
72
            (isBEPS, parm_dict) = parmsToDict(path_dict['parm_txt'])
Somnath, Suhas's avatar
Somnath, Suhas committed
73
        elif 'old_mat_parms' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
74
            parm_dict = self.__get_parms_from_old_mat(path_dict['old_mat_parms'])
75
76
            if parm_dict['VS_steps_per_full_cycle']==0: isBEPS=False
            else: isBEPS=True
Somnath, Suhas's avatar
Somnath, Suhas committed
77
        else:
78
            raise IOError('No parameters file found! Cannot translate this dataset!')
Unknown's avatar
Unknown committed
79

Somnath, Suhas's avatar
Somnath, Suhas committed
80
81
82
        ignored_plt_grps = []
        if isBEPS:
            parm_dict['data_type'] = 'BEPSData'
Unknown's avatar
Unknown committed
83

Somnath, Suhas's avatar
Somnath, Suhas committed
84
85
            field_mode = parm_dict['VS_measure_in_field_loops']
            std_expt = parm_dict['VS_mode'] != 'load user defined VS Wave from file'
Unknown's avatar
Unknown committed
86

Somnath, Suhas's avatar
Somnath, Suhas committed
87
            if not std_expt:
88
                raise ValueError('This translator does not handle user defined voltage spectroscopy')
Unknown's avatar
Unknown committed
89
90
91

            spec_label = getSpectroscopicParmLabel(parm_dict['VS_mode'])

Somnath, Suhas's avatar
Somnath, Suhas committed
92
            if parm_dict['VS_mode'] in ['DC modulation mode', 'current mode']:
Somnath, Suhas's avatar
Somnath, Suhas committed
93
94
95
96
97
98
99
100
101
102
103
                if field_mode == 'in and out-of-field':
                    tot_bins_multiplier = 2
                    udvs_denom = 1
                else:
                    if field_mode == 'out-of-field':
                        ignored_plt_grps = ['in-field']
                    else:
                        ignored_plt_grps = ['out-of-field']
            else:
                tot_bins_multiplier = 1
                udvs_denom = 1
Unknown's avatar
Unknown committed
104

Somnath, Suhas's avatar
Somnath, Suhas committed
105
106
107
        else:
            spec_label = 'None'
            parm_dict['data_type'] = 'BELineData'
Unknown's avatar
Unknown committed
108

Somnath, Suhas's avatar
Somnath, Suhas committed
109
110
        # Check file sizes:
        if 'read_real' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
111
112
            real_size = path.getsize(path_dict['read_real'])
            imag_size = path.getsize(path_dict['read_imag'])
Somnath, Suhas's avatar
Somnath, Suhas committed
113
114
115
        else:
            real_size = path.getsize(path_dict['write_real'])
            imag_size = path.getsize(path_dict['write_imag'])
Unknown's avatar
Unknown committed
116

Somnath, Suhas's avatar
Somnath, Suhas committed
117
118
119
        if real_size != imag_size:
            raise ValueError("Real and imaginary file sizes DON'T match!. Ending")

120
121
122
123
124
125
126
127
128
129
130
131
        #Check here if a second channel for current is present
        # Look for the file containing the current data

        file_names = listdir(folder_path)
        aux_files = []
        for fname in file_names:
            if 'AI2' in fname:
                if 'write' in fname:
                    current_file = path.join(folder_path, fname)
                    current_data_exists=True
                aux_files.append(path.join(folder_path, fname))

Unknown's avatar
Unknown committed
132
        add_pix = False
Somnath, Suhas's avatar
Somnath, Suhas committed
133
134
        num_rows = int(parm_dict['grid_num_rows'])
        num_cols = int(parm_dict['grid_num_cols'])
Unknown's avatar
Unknown committed
135
136
        num_pix = num_rows * num_cols
        tot_bins = real_size / (num_pix * 4)
Chris Smith's avatar
Chris Smith committed
137
        # Check for case where only a single pixel is missing.
Unknown's avatar
Unknown committed
138
139
140
        check_bins = real_size / ((num_pix - 1) * 4)

        if tot_bins % 1 and check_bins % 1:
141
            raise ValueError('Aborting! Some parameter appears to have changed in-between')
Somnath, Suhas's avatar
Somnath, Suhas committed
142
        elif not tot_bins % 1:
Chris Smith's avatar
Chris Smith committed
143
            # Everything's ok
Somnath, Suhas's avatar
Somnath, Suhas committed
144
145
146
            pass
        elif not check_bins % 1:
            tot_bins = check_bins
Unknown's avatar
Unknown committed
147
148
149
150
151
            warn('Warning:  A pixel seems to be missing from the data.  File will be padded with zeros.')
            add_pix = True

        tot_bins = int(tot_bins) * tot_bins_multiplier

Somnath, Suhas's avatar
Somnath, Suhas committed
152
        if 'parm_mat' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
153
            (bin_inds, bin_freqs, bin_FFT, ex_wfm) = self.__read_parms_mat(path_dict['parm_mat'], isBEPS)
Somnath, Suhas's avatar
Somnath, Suhas committed
154
        elif 'old_mat_parms' in path_dict.keys():
Somnath, Suhas's avatar
Somnath, Suhas committed
155
            (bin_inds, bin_freqs, bin_FFT, ex_wfm, dc_amp_vec) = self.__read_old_mat_be_vecs(path_dict['old_mat_parms'])
Somnath, Suhas's avatar
Somnath, Suhas committed
156
        else:
Unknown's avatar
Unknown committed
157
            band_width = parm_dict['BE_band_width_[Hz]'] * (0.5 - parm_dict['BE_band_edge_trim'])
Somnath, Suhas's avatar
Somnath, Suhas committed
158
            st_f = parm_dict['BE_center_frequency_[Hz]'] - band_width
Unknown's avatar
Unknown committed
159
            en_f = parm_dict['BE_center_frequency_[Hz]'] + band_width
Somnath, Suhas's avatar
Somnath, Suhas committed
160
            bin_freqs = np.linspace(st_f, en_f, tot_bins, dtype=np.float32)
Unknown's avatar
Unknown committed
161

162
            warn('No parms .mat file found.... Filling dummy values into ancillary datasets.')
Somnath, Suhas's avatar
Somnath, Suhas committed
163
164
165
            bin_inds = np.zeros(shape=tot_bins, dtype=np.int32)
            bin_FFT = np.zeros(shape=tot_bins, dtype=np.complex64)
            ex_wfm = np.zeros(shape=100, dtype=np.float32)
Unknown's avatar
Unknown committed
166

Somnath, Suhas's avatar
Somnath, Suhas committed
167
168
169
170
171
        # Forcing standardized datatypes:
        bin_inds = np.int32(bin_inds)
        bin_freqs = np.float32(bin_freqs)
        bin_FFT = np.complex64(bin_FFT)
        ex_wfm = np.float32(ex_wfm)
172

Somnath, Suhas's avatar
Somnath, Suhas committed
173
        self.FFT_BE_wave = bin_FFT
174

Somnath, Suhas's avatar
Somnath, Suhas committed
175
        if isBEPS:
Somnath, Suhas's avatar
Somnath, Suhas committed
176
            (UDVS_labs, UDVS_units, UDVS_mat) = self.__build_udvs_table(parm_dict)
Unknown's avatar
Unknown committed
177
178

            #             Remove the unused plot group columns before proceeding:
Somnath, Suhas's avatar
Somnath, Suhas committed
179
            (UDVS_mat, UDVS_labs, UDVS_units) = trimUDVS(UDVS_mat, UDVS_labs, UDVS_units, ignored_plt_grps)
Unknown's avatar
Unknown committed
180

181
            old_spec_inds = np.zeros(shape=(2, tot_bins), dtype=INDICES_DTYPE)
Unknown's avatar
Unknown committed
182
183
184
185
186

            #             Will assume that all excitation waveforms have same number of bins
            num_actual_udvs_steps = UDVS_mat.shape[0] / udvs_denom
            bins_per_step = tot_bins / num_actual_udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
187
            if bins_per_step % 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
188
189
                print('UDVS mat shape: {}, total bins: {}, bins per step: {}'.format(UDVS_mat.shape, tot_bins,
                                                                                     bins_per_step))
190
                raise ValueError('Non integer number of bins per step!')
Unknown's avatar
Unknown committed
191

Somnath, Suhas's avatar
Somnath, Suhas committed
192
193
            bins_per_step = int(bins_per_step)
            num_actual_udvs_steps = int(num_actual_udvs_steps)
Unknown's avatar
Unknown committed
194
195
196

            stind = 0
            for step_index in range(UDVS_mat.shape[0]):
Unknown's avatar
Unknown committed
197
198
199
                if UDVS_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                    continue
                # Bin step
200
                old_spec_inds[0, stind:stind + bins_per_step] = np.arange(bins_per_step, dtype=INDICES_DTYPE)
Unknown's avatar
Unknown committed
201
                # UDVS step
202
                old_spec_inds[1, stind:stind + bins_per_step] = step_index * np.ones(bins_per_step, dtype=INDICES_DTYPE)
Somnath, Suhas's avatar
Somnath, Suhas committed
203
                stind += bins_per_step
Somnath, Suhas's avatar
Somnath, Suhas committed
204
            del stind, step_index
Unknown's avatar
Unknown committed
205

Somnath, Suhas's avatar
Somnath, Suhas committed
206
        else:  # BE Line
Somnath, Suhas's avatar
Somnath, Suhas committed
207
            self.signal_type = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
208
            self.expt_type = 1  # Stephen has not used this index for some reason
Somnath, Suhas's avatar
Somnath, Suhas committed
209
210
            num_actual_udvs_steps = 1
            bins_per_step = tot_bins
Somnath, Suhas's avatar
Somnath, Suhas committed
211
            UDVS_labs = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod', 'be-line']
Somnath, Suhas's avatar
Somnath, Suhas committed
212
            UDVS_units = ['', 'V', 'A', '', '', '']
Somnath, Suhas's avatar
Somnath, Suhas committed
213
214
            UDVS_mat = np.array([1, 0, parm_dict['BE_amplitude_[V]'], 1, 1, 1],
                                dtype=np.float32).reshape(1, len(UDVS_labs))
Somnath, Suhas's avatar
Somnath, Suhas committed
215

Chris Smith's avatar
Chris Smith committed
216
217
            old_spec_inds = np.vstack((np.arange(tot_bins, dtype=INDICES_DTYPE),
                                       np.zeros(tot_bins, dtype=INDICES_DTYPE)))
Unknown's avatar
Unknown committed
218

Somnath, Suhas's avatar
Somnath, Suhas committed
219
220
221
        # Some very basic information that can help the processing / analysis crew
        parm_dict['num_bins'] = tot_bins
        parm_dict['num_pix'] = num_pix
222
        parm_dict['num_udvs_steps'] = num_actual_udvs_steps
Rama Vasudevan's avatar
Rama Vasudevan committed
223
        parm_dict['num_steps'] = num_actual_udvs_steps
Unknown's avatar
Unknown committed
224

Somnath, Suhas's avatar
Somnath, Suhas committed
225
        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
226
        for col_ind, col_name in enumerate(UDVS_labs):
Unknown's avatar
Unknown committed
227
228
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind + 1))

Somnath, Suhas's avatar
Somnath, Suhas committed
229
        # Need to add the Bin Waveform type - infer from UDVS        
Unknown's avatar
Unknown committed
230
        exec_bin_vec = self.signal_type * np.ones(len(bin_inds), dtype=np.int32)
Somnath, Suhas's avatar
Somnath, Suhas committed
231
232
233

        if self.expt_type == 2:
            # Need to double the vectors:
Unknown's avatar
Unknown committed
234
            exec_bin_vec = np.hstack((exec_bin_vec, -1 * exec_bin_vec))
Somnath, Suhas's avatar
Somnath, Suhas committed
235
236
            bin_inds = np.hstack((bin_inds, bin_inds))
            bin_freqs = np.hstack((bin_freqs, bin_freqs))
Somnath, Suhas's avatar
Somnath, Suhas committed
237
            # This is wrong but I don't know what else to do
Somnath, Suhas's avatar
Somnath, Suhas committed
238
            bin_FFT = np.hstack((bin_FFT, bin_FFT))
Unknown's avatar
Unknown committed
239

Somnath, Suhas's avatar
Somnath, Suhas committed
240
        # Create Spectroscopic Values and Spectroscopic Values Labels datasets
241
        # This is an old and legacy way of doing things. Ideally, all we would need ot do is just get the unit values
Somnath, Suhas's avatar
Somnath, Suhas committed
242
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_labs_names = createSpecVals(UDVS_mat,
243
                                                                                                     old_spec_inds,
Somnath, Suhas's avatar
Somnath, Suhas committed
244
245
246
247
248
                                                                                                     bin_freqs,
                                                                                                     exec_bin_vec,
                                                                                                     parm_dict,
                                                                                                     UDVS_labs,
                                                                                                     UDVS_units)
249
250
251
252
        # Not sure what is happening here but this should work.
        spec_dim_dict = dict()
        for entry in spec_vals_labs_names:
            spec_dim_dict[entry[0] + '_parameters'] = entry[1]
Chris Smith's avatar
Chris Smith committed
253

Somnath, Suhas's avatar
Somnath, Suhas committed
254
        spec_vals_slices = dict()
Unknown's avatar
Unknown committed
255
256
257
        #         if len(spec_vals_labs) == 1:
        #             spec_vals_slices[spec_vals_labs[0]]=(slice(0,1,None),)
        #         else:
Somnath, Suhas's avatar
Somnath, Suhas committed
258
259

        for row_ind, row_name in enumerate(spec_vals_labs):
Unknown's avatar
Unknown committed
260
            spec_vals_slices[row_name] = (slice(row_ind, row_ind + 1), slice(None))
Somnath, Suhas's avatar
Somnath, Suhas committed
261

262
263
        if path.exists(h5_path):
            remove(h5_path)
Chris Smith's avatar
Chris Smith committed
264

265
266
        # First create the file
        h5_f = h5py.File(h5_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
267

268
        # Then write root level attributes
269
        global_parms = generate_dummy_main_parms()
Somnath, Suhas's avatar
Somnath, Suhas committed
270
271
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
272
273
274
275
        try:
            global_parms['experiment_date'] = parm_dict['File_date_and_time']
        except KeyError:
            global_parms['experiment_date'] = '1:1:1'
Chris Smith's avatar
Chris Smith committed
276

Somnath, Suhas's avatar
Somnath, Suhas committed
277
        # assuming that the experiment was completed:
Unknown's avatar
Unknown committed
278
279
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
Somnath, Suhas's avatar
Somnath, Suhas committed
280
        global_parms['data_type'] = parm_dict['data_type']
Somnath, Suhas's avatar
Somnath, Suhas committed
281
        global_parms['translator'] = 'ODF'
282
        write_simple_attrs(h5_f, global_parms)
283
        write_book_keeping_attrs(h5_f)
Unknown's avatar
Unknown committed
284

285
286
        # Then create the measurement group
        h5_meas_group = create_indexed_group(h5_f, 'Measurement')
Unknown's avatar
Unknown committed
287

288
289
        # Write attributes at the measurement group level
        write_simple_attrs(h5_meas_group, parm_dict)
Unknown's avatar
Unknown committed
290

291
292
        # Create the Channel group
        h5_chan_grp = create_indexed_group(h5_meas_group, 'Channel')
Unknown's avatar
Unknown committed
293

294
        # Write channel group attributes
Rama Vasudevan's avatar
Rama Vasudevan committed
295
296
        write_simple_attrs(h5_chan_grp, {'Channel_Input': 'IO_Analog_Input_1',
                                         'channel_type': 'BE'})
Unknown's avatar
Unknown committed
297

298
        # Now the datasets!
Chris Smith's avatar
Chris Smith committed
299
        h5_chan_grp.create_dataset('Excitation_Waveform', data=ex_wfm)
Unknown's avatar
Unknown committed
300

301
302
303
304
305
        h5_udvs = h5_chan_grp.create_dataset('UDVS', data=UDVS_mat)
        write_region_references(h5_udvs, udvs_slices, add_labels_attr=True, verbose=verbose)
        write_simple_attrs(h5_udvs, {'units': UDVS_units}, verbose=verbose)
        
        # ds_udvs_labs = MicroDataset('UDVS_Labels',np.array(UDVS_labs))
Chris Smith's avatar
Chris Smith committed
306
        h5_chan_grp.create_dataset('UDVS_Indices', data=old_spec_inds[1])
307
308

        # ds_spec_labs = MicroDataset('Spectroscopic_Labels',np.array(['Bin','UDVS_Step']))
Chris Smith's avatar
Chris Smith committed
309
310
        h5_chan_grp.create_dataset('Bin_Step', data=np.arange(bins_per_step, dtype=INDICES_DTYPE),
                                   dtype=INDICES_DTYPE)
311

Chris Smith's avatar
Chris Smith committed
312
313
314
315
        h5_chan_grp.create_dataset('Bin_Indices', data=bin_inds, dtype=INDICES_DTYPE)
        h5_chan_grp.create_dataset('Bin_Frequencies', data=bin_freqs)
        h5_chan_grp.create_dataset('Bin_FFT', data=bin_FFT)
        h5_chan_grp.create_dataset('Bin_Wfm_Type', data=exec_bin_vec)
316
317
318
319
320
321
322
323
324

        pos_dims = [Dimension('X', 'm', np.arange(num_cols)), Dimension('Y', 'm', np.arange(num_rows))]
        h5_pos_ind, h5_pos_val = write_ind_val_dsets(h5_chan_grp, pos_dims, is_spectral=False, verbose=verbose)

        h5_spec_inds = h5_chan_grp.create_dataset('Spectroscopic_Indices', data=spec_inds, dtype=INDICES_DTYPE)        
        h5_spec_vals = h5_chan_grp.create_dataset('Spectroscopic_Values', data=np.array(spec_vals), dtype=VALUES_DTYPE)
        for dset in [h5_spec_inds, h5_spec_vals]:
            write_region_references(dset, spec_vals_slices, add_labels_attr=True, verbose=verbose)
            write_simple_attrs(dset, {'units': spec_vals_units}, verbose=verbose)
325
            write_simple_attrs(dset, spec_dim_dict)
326
327

        # Noise floor should be of shape: (udvs_steps x 3 x positions)
Chris Smith's avatar
Chris Smith committed
328
329
        h5_chan_grp.create_dataset('Noise_Floor', (num_pix, num_actual_udvs_steps), dtype=nf32,
                                   chunks=(1, num_actual_udvs_steps))
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344

        """
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares
        of UDVS steps by pixels.  N is determined dynamically based on the dimensions of the
        dataset.  Currently it is set such that individual chunks are less than 10kB in size.

        Chris Smith -- csmith55@utk.edu
        """
        BEPS_chunks = calc_chunks([num_pix, tot_bins],
                                  np.complex64(0).itemsize,
                                  unit_chunks=(1, bins_per_step))
        self.h5_raw = write_main_dataset(h5_chan_grp, (num_pix, tot_bins), 'Raw_Data', 'Piezoresponse', 'V', None, None,
                                         dtype=np.complex64, chunks=BEPS_chunks, compression='gzip',
                                         h5_pos_inds=h5_pos_ind, h5_pos_vals=h5_pos_val, h5_spec_inds=h5_spec_inds,
                                         h5_spec_vals=h5_spec_vals, verbose=verbose)
Somnath, Suhas's avatar
Somnath, Suhas committed
345

Chris Smith's avatar
Chris Smith committed
346
        self._read_data(UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix)
Unknown's avatar
Unknown committed
347

348
        generatePlotGroups(self.h5_raw, self.mean_resp, folder_path, basename,
Somnath, Suhas's avatar
Somnath, Suhas committed
349
                           self.max_resp, self.min_resp, max_mem_mb=self.max_ram,
Somnath, Suhas's avatar
Somnath, Suhas committed
350
                           spec_label=spec_label, show_plots=show_plots, save_plots=save_plots,
Unknown's avatar
Unknown committed
351
                           do_histogram=do_histogram, debug=verbose)
Unknown's avatar
Unknown committed
352

353
354
355
356
357
        self.h5_raw = USIDataset(self.h5_raw)
        #Go ahead and read the current data in the second (current) channel
        if current_data_exists: #If a .dat file matches
            self._read_secondary_channel(h5_meas_group, aux_files)

358
        h5_f.close()
Unknown's avatar
Unknown committed
359

Somnath, Suhas's avatar
Somnath, Suhas committed
360
        return h5_path
Chris Smith's avatar
Chris Smith committed
361

362

Chris Smith's avatar
Chris Smith committed
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
    def _read_data(self, UDVS_mat, parm_dict, path_dict, real_size, isBEPS, add_pix):
        """
        Checks if the data is BEPS or BELine and calls the correct function to read the data from
        file

        Parameters
        ----------
        UDVS_mat : numpy.ndarray of float
            UDVS table
        parm_dict : dict
            Experimental parameters
        path_dict : dict
            Dictionary of data files to be read
        real_size : dict
            Size of each data file
        isBEPS : boolean
            Is the data BEPS
        add_pix : boolean
            Does the reader need to add extra pixels to the end of the dataset

        Returns
        -------
        None
        """
        # Now read the raw data files:
        if not isBEPS:
            # Do this for all BE-Line (always small enough to read in one shot)
390
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
391
392
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'out-of-field':
            # Do this for out-of-field BEPS ONLY that is also small (256 MB)
393
            self.__quick_read_data(path_dict['read_real'], path_dict['read_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
394
395
        elif real_size < self.max_ram and parm_dict['VS_measure_in_field_loops'] == 'in-field':
            # Do this for in-field only
396
            self.__quick_read_data(path_dict['write_real'], path_dict['write_imag'], parm_dict['num_udvs_steps'])
Chris Smith's avatar
Chris Smith committed
397
398
        else:
            # Large BEPS datasets OR those with in-and-out of field
Somnath, Suhas's avatar
Somnath, Suhas committed
399
            self.__read_beps_data(path_dict, UDVS_mat.shape[0], parm_dict['VS_measure_in_field_loops'], add_pix)
400
        self.h5_raw.file.flush()
Chris Smith's avatar
Chris Smith committed
401

Somnath, Suhas's avatar
Somnath, Suhas committed
402
    def __read_beps_data(self, path_dict, udvs_steps, mode, add_pixel=False):
Somnath, Suhas's avatar
Somnath, Suhas committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
        """
        Reads the imaginary and real data files pixelwise and writes to the H5 file 
        
        Parameters 
        --------------------
        path_dict : dictionary
            Dictionary containing the absolute paths of the real and imaginary data files
        udvs_steps : unsigned int
            Number of UDVS steps
        mode : String / Unicode
            'in-field', 'out-of-field', or 'in and out-of-field'
        add_pixel : boolean. (Optional; default is False)
            If an empty pixel worth of data should be written to the end             
        
        Returns 
        -------------------- 
        None
        """
Unknown's avatar
Unknown committed
421

Somnath, Suhas's avatar
Somnath, Suhas committed
422
        print('---- reading pixel-by-pixel ----------')
Unknown's avatar
Unknown committed
423
424
425
426

        bytes_per_pix = self.h5_raw.shape[1] * 4
        step_size = self.h5_raw.shape[1] / udvs_steps

Somnath, Suhas's avatar
Somnath, Suhas committed
427
        if mode == 'out-of-field':
Unknown's avatar
Unknown committed
428
            parsers = [BEodfParser(path_dict['read_real'], path_dict['read_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
429
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
430
        elif mode == 'in-field':
Unknown's avatar
Unknown committed
431
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
432
                                   self.h5_raw.shape[0], bytes_per_pix)]
Somnath, Suhas's avatar
Somnath, Suhas committed
433
434
        elif mode == 'in and out-of-field':
            # each file will only have half the udvs steps:
Unknown's avatar
Unknown committed
435
            if 0.5 * udvs_steps % 1:
436
437
                raise ValueError('Odd number of UDVS')

Unknown's avatar
Unknown committed
438
            udvs_steps = int(0.5 * udvs_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
439
            # be careful - each pair contains only half the necessary bins - so read half
Unknown's avatar
Unknown committed
440
            parsers = [BEodfParser(path_dict['write_real'], path_dict['write_imag'],
Somnath, Suhas's avatar
Somnath, Suhas committed
441
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2)),
Unknown's avatar
Unknown committed
442
443
444
                       BEodfParser(path_dict['read_real'], path_dict['read_imag'],
                                   self.h5_raw.shape[0], int(bytes_per_pix / 2))]

Somnath, Suhas's avatar
Somnath, Suhas committed
445
            if step_size % 1:
446
447
                raise ValueError('strange number of bins per UDVS step. Exiting')

Somnath, Suhas's avatar
Somnath, Suhas committed
448
            step_size = int(step_size)
449

450
451
        rand_spectra = self.__get_random_spectra(parsers, self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
452
        take_conjugate = requires_conjugate(rand_spectra)
453

Somnath, Suhas's avatar
Somnath, Suhas committed
454
455
456
457
        self.mean_resp = np.zeros(shape=(self.h5_raw.shape[1]), dtype=np.complex64)
        self.max_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)
        self.min_resp = np.zeros(shape=(self.h5_raw.shape[0]), dtype=np.float32)

Unknown's avatar
Unknown committed
458
        numpix = self.h5_raw.shape[0]
Somnath, Suhas's avatar
Somnath, Suhas committed
459
460
461
        """ 
        Don't try to do the last step if a pixel is missing.   
        This will be handled after the loop. 
Unknown's avatar
Unknown committed
462
463
464
465
        """
        if add_pixel:
            numpix -= 1

Somnath, Suhas's avatar
Somnath, Suhas committed
466
        for pix_indx in range(numpix):
Somnath, Suhas's avatar
Somnath, Suhas committed
467
            if self.h5_raw.shape[0] > 5:
Unknown's avatar
Unknown committed
468
469
470
                if pix_indx % int(round(self.h5_raw.shape[0] / 10)) == 0:
                    print('Reading... {} complete'.format(round(100 * pix_indx / self.h5_raw.shape[0])))

Somnath, Suhas's avatar
Somnath, Suhas committed
471
472
473
            # get the raw stream from each parser
            pxl_data = list()
            for prsr in parsers:
Somnath, Suhas's avatar
Somnath, Suhas committed
474
                pxl_data.append(prsr.read_pixel())
Unknown's avatar
Unknown committed
475

Somnath, Suhas's avatar
Somnath, Suhas committed
476
477
478
479
480
            # interleave if both in and out of field
            # we are ignoring user defined possibilities...
            if mode == 'in and out-of-field':
                in_fld = pxl_data[0]
                out_fld = pxl_data[1]
Unknown's avatar
Unknown committed
481

Somnath, Suhas's avatar
Somnath, Suhas committed
482
483
                in_fld_2 = in_fld.reshape(udvs_steps, step_size)
                out_fld_2 = out_fld.reshape(udvs_steps, step_size)
Unknown's avatar
Unknown committed
484
                raw_mat = np.empty((udvs_steps * 2, step_size), dtype=out_fld.dtype)
Somnath, Suhas's avatar
Somnath, Suhas committed
485
486
                raw_mat[0::2, :] = in_fld_2
                raw_mat[1::2, :] = out_fld_2
Somnath, Suhas's avatar
Somnath, Suhas committed
487
488
                raw_vec = raw_mat.reshape(in_fld.size + out_fld.size).transpose()
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
489
                raw_vec = pxl_data[0]  # only one parser
Somnath, Suhas's avatar
Somnath, Suhas committed
490
491
            self.max_resp[pix_indx] = np.max(np.abs(raw_vec))
            self.min_resp[pix_indx] = np.min(np.abs(raw_vec))
Unknown's avatar
Unknown committed
492
            self.mean_resp = (1 / (pix_indx + 1)) * (raw_vec + pix_indx * self.mean_resp)
493
494
495

            if take_conjugate:
                raw_vec = np.conjugate(raw_vec)
496
            self.h5_raw[pix_indx, :] = np.complex64(raw_vec[:])
497
            self.h5_raw.file.flush()
Unknown's avatar
Unknown committed
498

Somnath, Suhas's avatar
Somnath, Suhas committed
499
        # Add zeros to main_data for the missing pixel. 
Unknown's avatar
Unknown committed
500
501
502
        if add_pixel:
            self.h5_raw[-1, :] = 0 + 0j

Somnath, Suhas's avatar
Somnath, Suhas committed
503
        print('---- Finished reading files -----')
504
505

    def __quick_read_data(self, real_path, imag_path, udvs_steps):
Somnath, Suhas's avatar
Somnath, Suhas committed
506
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
507
508
509
510
511
512
513
514
        Returns information about the excitation BE waveform present in the .mat file

        Parameters
        -----------
        real_path : String / Unicode
            Absolute file path of the real data file
        imag_path : String / Unicode
            Absolute file path of the real data file
515
516
        udvs_steps : unsigned int
            Number of UDVS steps
Somnath, Suhas's avatar
Somnath, Suhas committed
517
        """
Unknown's avatar
Unknown committed
518
        print('---- reading all data at once ----------')
Somnath, Suhas's avatar
Somnath, Suhas committed
519

Unknown's avatar
Unknown committed
520
        parser = BEodfParser(real_path, imag_path, self.h5_raw.shape[0], self.h5_raw.shape[1] * 4)
521
522

        step_size = self.h5_raw.shape[1] / udvs_steps
523
524
        rand_spectra = self.__get_random_spectra([parser], self.h5_raw.shape[0], udvs_steps, step_size,
                                                 num_spectra=self.num_rand_spectra)
525
        take_conjugate = requires_conjugate(rand_spectra)
Somnath, Suhas's avatar
Somnath, Suhas committed
526
        raw_vec = parser.read_all_data()
527
        if take_conjugate:
528
            print('Taking conjugate to ensure positive Quality factors')
529
            raw_vec = np.conjugate(raw_vec)
Unknown's avatar
Unknown committed
530

Somnath, Suhas's avatar
Somnath, Suhas committed
531
        raw_mat = raw_vec.reshape(self.h5_raw.shape[0], self.h5_raw.shape[1])
Unknown's avatar
Unknown committed
532

Somnath, Suhas's avatar
Somnath, Suhas committed
533
        # Write to the h5 dataset:
Somnath, Suhas's avatar
Somnath, Suhas committed
534
535
536
        self.mean_resp = np.mean(raw_mat, axis=0)
        self.max_resp = np.amax(np.abs(raw_mat), axis=0)
        self.min_resp = np.amin(np.abs(raw_mat), axis=0)
537
        self.h5_raw[:, :] = np.complex64(raw_mat)
538
        self.h5_raw.file.flush()
Somnath, Suhas's avatar
Somnath, Suhas committed
539

Unknown's avatar
Unknown committed
540
541
        print('---- Finished reading files -----')

542
    def _parse_file_path(self, data_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
543
544
545
546
547
548
549
        """
        Returns the basename and a dictionary containing the absolute file paths for the
        real and imaginary data files, text and mat parameter files in a dictionary
        
        Parameters 
        --------------------
        data_filepath: String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
550
            Absolute path of any file in the same directory as the .dat files
Somnath, Suhas's avatar
Somnath, Suhas committed
551
552
553
554
555
556
557
558
559
        
        Returns 
        --------------------
        basename : String / Unicode
            Basename of the dataset      
        path_dict : Dictionary
            Dictionary containing absolute paths of all necessary data and parameter files
        """
        (folder_path, basename) = path.split(data_filepath)
Unknown's avatar
Unknown committed
560
        (super_folder, basename) = path.split(folder_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
561

562
563
        if basename.endswith('_d') or basename.endswith('_c'):
            # Old old data format where the folder ended with a _d or _c to denote a completed spectroscopic run
Somnath, Suhas's avatar
Somnath, Suhas committed
564
565
566
567
568
569
570
571
            basename = basename[:-2]
        """
        A single pair of real and imaginary files are / were generated for:
            BE-Line and BEPS (compiled version only generated out-of-field or 'read')
        Two pairs of real and imaginary files were generated for later BEPS datasets
            These have 'read' and 'write' prefixes to denote out or in field respectively
        """
        path_dict = dict()
Unknown's avatar
Unknown committed
572

Somnath, Suhas's avatar
Somnath, Suhas committed
573
        for file_name in listdir(folder_path):
Chris Smith's avatar
Chris Smith committed
574
            abs_path = path.join(folder_path, file_name)
Somnath, Suhas's avatar
Somnath, Suhas committed
575
576
577
578
579
            if file_name.endswith('.txt') and file_name.find('parm') > 0:
                path_dict['parm_txt'] = abs_path
            elif file_name.find('.mat') > 0:
                if file_name.find('more_parms') > 0:
                    path_dict['parm_mat'] = abs_path
Unknown's avatar
Unknown committed
580
                elif file_name == (basename + '.mat'):
Somnath, Suhas's avatar
Somnath, Suhas committed
581
582
583
584
585
586
587
588
589
590
591
592
                    path_dict['old_mat_parms'] = abs_path
            elif file_name.endswith('.dat'):
                # Need to account for the second AI channel here
                file_tag = 'read'
                if file_name.find('write') > 0:
                    file_tag = 'write'
                if file_name.find('real') > 0:
                    file_tag += '_real'
                elif file_name.find('imag') > 0:
                    file_tag += '_imag'
                path_dict[file_tag] = abs_path

Chris Smith's avatar
Chris Smith committed
593
        return basename, path_dict
Somnath, Suhas's avatar
Somnath, Suhas committed
594

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
    def _read_secondary_channel(self, h5_meas_group, aux_file_path):
        """
        Reads secondary channel stored in AI .mat file
        Currently works for in-field measurements only, but should be updated to
        include both in and out of field measurements

        Parameters
        -----------
        h5_meas_group : h5 group
            Reference to the Measurement group
        aux_file_path : String / Unicode
            Absolute file path of the secondary channel file.
        """
        print('---- Reading Secondary Channel  ----------')
        if len(aux_file_path)>1:
            print('Detected multiple files, assuming in and out of field')
            aux_file_paths = aux_file_path
        else:
            aux_file_paths = list(aux_file_path)

        freq_index = self.h5_raw.spec_dim_labels.index('Frequency')
        num_pix = self.h5_raw.shape[0]
        spectral_len = 1

        for i in range(len(self.h5_raw.spec_dim_sizes)):
            if i == freq_index:
                continue
            spectral_len = spectral_len * self.h5_raw.spec_dim_sizes[i]

        #num_forc_cycles = self.h5_raw.spec_dim_sizes[self.h5_raw.spec_dim_labels.index("FORC")]
        #num_dc_steps =  self.h5_raw.spec_dim_sizes[self.h5_raw.spec_dim_labels.index("DC_Offset")]

        # create a new channel
        h5_current_channel_group = create_indexed_group(h5_meas_group, 'Channel')

        # Copy attributes from the main channel
        copy_attributes(self.h5_raw.parent, h5_current_channel_group)

        # Modify attributes that are different
        write_simple_attrs(h5_current_channel_group, {'Channel_Input': 'IO_Analog_Input_2',
                                                      'channel_type': 'Current'}, verbose=True)

        #Get the reduced dimensions
        h5_current_spec_inds, h5_current_spec_values = write_reduced_spec_dsets(h5_current_channel_group,
                                                        self.h5_raw.h5_spec_inds,
                                                        self.h5_raw.h5_spec_vals, 'Frequency')


        h5_current_main = write_main_dataset(h5_current_channel_group,  # parent HDF5 group
                                             (num_pix, spectral_len),  # shape of Main dataset
                                             'Raw_Data',  # Name of main dataset
                                             'Current',  # Physical quantity contained in Main dataset
                                             'nA',  # Units for the physical quantity
                                             None,  # Position dimensions
                                             None,  # Spectroscopic dimensions
                                             h5_pos_inds=self.h5_raw.h5_pos_inds,
                                             h5_pos_vals=self.h5_raw.h5_pos_vals,
                                             h5_spec_inds=h5_current_spec_inds,
                                             h5_spec_vals=h5_current_spec_values,
                                             dtype=np.float32,  # data type / precision
                                             main_dset_attrs={'IO_rate': 4E+6, 'Amplifier_Gain': 9})

        # Now calculate the number of positions that can be stored in memory in one go.
        b_per_position = np.float32(0).itemsize * spectral_len

        max_pos_per_read = int(np.floor((get_available_memory()) / b_per_position))

        # if self._verbose:
        print('Allowed to read {} pixels per chunk'.format(max_pos_per_read))

        #Open the read and write files and write them to the hdf5 file
        for aux_file in aux_file_paths:
            if 'write' in aux_file:
                infield = True
            else:
                infield=False

            cur_file = open(aux_file, "rb")

            start_pix = 0

            while start_pix < num_pix:
                end_pix = min(num_pix, start_pix + max_pos_per_read)

                # TODO: Fix for when it won't fit in memory.

                #if max_pos_per_read * b_per_position > num_pix * b_per_position:
                cur_data = np.frombuffer(cur_file.read(), dtype='f')
                #else:
                #cur_data = np.frombuffer(cur_file.read(max_pos_per_read * b_per_position), dtype='f')

                cur_data = cur_data.reshape(end_pix - start_pix, spectral_len//2)

                # Write to h5
                if infield:
                    h5_current_main[start_pix:end_pix, ::2] = cur_data
                else:
                    h5_current_main[start_pix:end_pix, 1::2] = cur_data
                start_pix = end_pix


Somnath, Suhas's avatar
Somnath, Suhas committed
696
697
    @staticmethod
    def __read_old_mat_be_vecs(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
        """
        Returns information about the excitation BE waveform present in the 
        more parms.mat file
        
        Parameters 
        --------------------
        filepath : String or unicode
            Absolute filepath of the .mat parameter file
        
        Returns 
        --------------------
        bin_inds : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        bin_w : 1D numpy float array
            Excitation bin Frequencies
        bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        BE_wave : 1D numpy float array
            Band Excitation waveform
        dc_amp_vec_full : 1D numpy float array
            spectroscopic waveform. 
            This information will be necessary for fixing the UDVS for AC modulation for example
        """
Unknown's avatar
Unknown committed
721
        matread = loadmat(file_path, squeeze_me=True)
Somnath, Suhas's avatar
Somnath, Suhas committed
722
        BE_wave = matread['BE_wave']
Unknown's avatar
Unknown committed
723
        bin_inds = matread['bin_ind'] - 1  # Python base 0
Somnath, Suhas's avatar
Somnath, Suhas committed
724
725
726
        bin_w = matread['bin_w']
        dc_amp_vec_full = matread['dc_amp_vec_full']
        FFT_full = np.fft.fftshift(np.fft.fft(BE_wave))
Somnath, Suhas's avatar
Somnath, Suhas committed
727
728
        bin_FFT = np.conjugate(FFT_full[bin_inds])
        return bin_inds, bin_w, bin_FFT, BE_wave, dc_amp_vec_full
Unknown's avatar
Unknown committed
729

Somnath, Suhas's avatar
Somnath, Suhas committed
730
731
    @staticmethod
    def __get_parms_from_old_mat(file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
        """
        Formats parameters found in the old parameters .mat file into a dictionary
        as though the dataset had a parms.txt describing it
        
        Parameters 
        --------------------
        file_path : Unicode / String
            absolute filepath of the .mat file containing the parameters
            
        Returns 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        """
        parm_dict = dict()
        matread = loadmat(file_path, squeeze_me=True)
Unknown's avatar
Unknown committed
748
749
750

        parm_dict['IO_rate'] = str(int(matread['AO_rate'] / 1E+6)) + ' MHz'

Somnath, Suhas's avatar
Somnath, Suhas committed
751
752
753
754
755
        position_vec = matread['position_vec']
        parm_dict['grid_current_row'] = position_vec[0]
        parm_dict['grid_current_col'] = position_vec[1]
        parm_dict['grid_num_rows'] = position_vec[2]
        parm_dict['grid_num_cols'] = position_vec[3]
Unknown's avatar
Unknown committed
756

Somnath, Suhas's avatar
Somnath, Suhas committed
757
758
        if position_vec[0] != position_vec[1] or position_vec[2] != position_vec[3]:
            warn('WARNING: Incomplete dataset. Translation not guaranteed!')
Somnath, Suhas's avatar
Somnath, Suhas committed
759
            parm_dict['grid_num_rows'] = position_vec[0]  # set to number of present cols and rows
Somnath, Suhas's avatar
Somnath, Suhas committed
760
            parm_dict['grid_num_cols'] = position_vec[1]
Unknown's avatar
Unknown committed
761

Somnath, Suhas's avatar
Somnath, Suhas committed
762
763
764
765
766
767
768
769
770
        BE_parm_vec_1 = matread['BE_parm_vec_1']
        # Not required for translation but necessary to have
        if BE_parm_vec_1[0] == 3:
            parm_dict['BE_phase_content'] = 'chirp-sinc hybrid'
        else:
            parm_dict['BE_phase_content'] = 'Unknown'
        parm_dict['BE_center_frequency_[Hz]'] = BE_parm_vec_1[1]
        parm_dict['BE_band_width_[Hz]'] = BE_parm_vec_1[2]
        parm_dict['BE_amplitude_[V]'] = BE_parm_vec_1[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
771
772
        parm_dict['BE_band_edge_smoothing_[s]'] = BE_parm_vec_1[4]  # 150 most likely
        parm_dict['BE_phase_variation'] = BE_parm_vec_1[5]  # 0.01 most likely
Unknown's avatar
Unknown committed
773
774
775
        parm_dict['BE_window_adjustment'] = BE_parm_vec_1[6]
        parm_dict['BE_points_per_step'] = 2 ** int(BE_parm_vec_1[7])
        parm_dict['BE_repeats'] = 2 ** int(BE_parm_vec_1[8])
Somnath, Suhas's avatar
Somnath, Suhas committed
776
777
778
779
        try:
            parm_dict['BE_bins_per_read'] = matread['bins_per_band_s']
        except KeyError:
            parm_dict['BE_bins_per_read'] = len(matread['bin_w'])
Unknown's avatar
Unknown committed
780

Somnath, Suhas's avatar
Somnath, Suhas committed
781
        assembly_parm_vec = matread['assembly_parm_vec']
Unknown's avatar
Unknown committed
782

Somnath, Suhas's avatar
Somnath, Suhas committed
783
784
785
786
787
788
        if assembly_parm_vec[2] == 0:
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
        elif assembly_parm_vec[2] == 1:
            parm_dict['VS_measure_in_field_loops'] = 'in and out-of-field'
        else:
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
Unknown's avatar
Unknown committed
789

Somnath, Suhas's avatar
Somnath, Suhas committed
790
791
792
793
794
        parm_dict['IO_Analog_Input_1'] = '+/- 10V, FFT'
        if assembly_parm_vec[3] == 0:
            parm_dict['IO_Analog_Input_2'] = 'off'
        else:
            parm_dict['IO_Analog_Input_2'] = '+/- 10V, FFT'
Unknown's avatar
Unknown committed
795

Somnath, Suhas's avatar
Somnath, Suhas committed
796
797
        # num_driving_bands = assembly_parm_vec[0]  # 0 = 1, 1 = 2 bands
        # band_combination_order = assembly_parm_vec[1]  # 0 parallel 1 series
Unknown's avatar
Unknown committed
798

Somnath, Suhas's avatar
Somnath, Suhas committed
799
800
        VS_parms = matread['SS_parm_vec']
        dc_amp_vec_full = matread['dc_amp_vec_full']
Unknown's avatar
Unknown committed
801
802
803
804

        VS_start_V = VS_parms[4]
        VS_start_loop_amp = VS_parms[5]
        VS_final_loop_amp = VS_parms[6]
Somnath, Suhas's avatar
Somnath, Suhas committed
805
        # VS_read_write_ratio = VS_parms[8]  # 1 <- SS_read_write_ratio
Unknown's avatar
Unknown committed
806

Somnath, Suhas's avatar
Somnath, Suhas committed
807
        parm_dict['VS_set_pulse_amplitude_[V]'] = VS_parms[9]  # 0 <- SS_set_pulse_amp
Unknown's avatar
Unknown committed
808
        parm_dict['VS_read_voltage_[V]'] = VS_parms[3]
Somnath, Suhas's avatar
Somnath, Suhas committed
809
810
        parm_dict['VS_steps_per_full_cycle'] = VS_parms[7]
        parm_dict['VS_cycle_fraction'] = 'full'
Unknown's avatar
Unknown committed
811
        parm_dict['VS_cycle_phase_shift'] = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
812
        parm_dict['VS_number_of_cycles'] = VS_parms[2]
Somnath, Suhas's avatar
Somnath, Suhas committed
813
814
815
816
817
        parm_dict['FORC_num_of_FORC_cycles'] = 1
        parm_dict['FORC_V_high1_[V]'] = 0
        parm_dict['FORC_V_high2_[V]'] = 0
        parm_dict['FORC_V_low1_[V]'] = 0
        parm_dict['FORC_V_low2_[V]'] = 0
Unknown's avatar
Unknown committed
818

Somnath, Suhas's avatar
Somnath, Suhas committed
819
820
        if VS_parms[0] == 0:
            parm_dict['VS_mode'] = 'DC modulation mode'
Unknown's avatar
Unknown committed
821
822
823
            parm_dict['VS_amplitude_[V]'] = 0.5 * (
                max(dc_amp_vec_full) - min(dc_amp_vec_full))  # SS_max_offset_amplitude
            parm_dict['VS_offset_[V]'] = max(dc_amp_vec_full) + min(dc_amp_vec_full)
Somnath, Suhas's avatar
Somnath, Suhas committed
824
825
826
        elif VS_parms[0] == 1:
            # FORC
            parm_dict['VS_mode'] = 'DC modulation mode'
Somnath, Suhas's avatar
Somnath, Suhas committed
827
            parm_dict['VS_amplitude_[V]'] = 1  # VS_parms[1] # SS_max_offset_amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
828
            parm_dict['VS_offset_[V]'] = 0
Unknown's avatar
Unknown committed
829
            parm_dict['VS_number_of_cycles'] = 1
Somnath, Suhas's avatar
Somnath, Suhas committed
830
831
832
833
834
            parm_dict['FORC_num_of_FORC_cycles'] = VS_parms[2]
            parm_dict['FORC_V_high1_[V]'] = VS_start_V
            parm_dict['FORC_V_high2_[V]'] = VS_start_V
            parm_dict['FORC_V_low1_[V]'] = VS_start_V - VS_start_loop_amp
            parm_dict['FORC_V_low2_[V]'] = VS_start_V - VS_final_loop_amp
Somnath, Suhas's avatar
Somnath, Suhas committed
835
836
837
        elif VS_parms[0] == 2:
            # AC mode 
            parm_dict['VS_mode'] = 'AC modulation mode with time reversal'
Somnath, Suhas's avatar
Somnath, Suhas committed
838
839
            parm_dict['VS_amplitude_[V]'] = 0.5 * VS_final_loop_amp
            parm_dict['VS_offset_[V]'] = 0  # this is not correct. Fix manually when it comes to UDVS generation?
Somnath, Suhas's avatar
Somnath, Suhas committed
840
841
        else:
            parm_dict['VS_mode'] = 'Custom'
Unknown's avatar
Unknown committed
842

Somnath, Suhas's avatar
Somnath, Suhas committed
843
        return parm_dict
Unknown's avatar
Unknown committed
844

Somnath, Suhas's avatar
Somnath, Suhas committed
845
846
    @staticmethod
    def __read_parms_mat(file_path, is_beps):
Somnath, Suhas's avatar
Somnath, Suhas committed
847
848
849
850
851
        """
        Returns information about the excitation BE waveform present in the more parms.mat file
        
        Parameters 
        --------------------
Somnath, Suhas's avatar
Somnath, Suhas committed
852
        file_path : String / Unicode
Somnath, Suhas's avatar
Somnath, Suhas committed
853
            Absolute filepath of the .mat parameter file
Somnath, Suhas's avatar
Somnath, Suhas committed
854
        is_beps : Boolean
Somnath, Suhas's avatar
Somnath, Suhas committed
855
856
857
858
859
860
861
862
863
864
865
866
867
            Whether or not this is BEPS or BE-Line
        
        Returns 
        --------------------
        BE_bin_ind : 1D numpy unsigned int array
            Indices of the excited and measured frequency bins
        BE_bin_w : 1D numpy float array
            Excitation bin Frequencies
        BE_bin_FFT : 1D numpy complex array
            FFT of the BE waveform for the excited bins
        ex_wfm : 1D numpy float array
            Band Excitation waveform
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
868
        if not path.exists(file_path):
869
            raise IOError('NO "More parms" file found')
Somnath, Suhas's avatar
Somnath, Suhas committed
870
        if is_beps:
Somnath, Suhas's avatar
Somnath, Suhas committed
871
872
873
            fft_name = 'FFT_BE_wave'
        else:
            fft_name = 'FFT_BE_rev_wave'
Somnath, Suhas's avatar
Somnath, Suhas committed
874
        matread = loadmat(file_path, variable_names=['BE_bin_ind', 'BE_bin_w', fft_name])
Unknown's avatar
Unknown committed
875
        BE_bin_ind = np.squeeze(matread['BE_bin_ind']) - 1  # From Matlab (base 1) to Python (base 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
876
877
        BE_bin_w = np.squeeze(matread['BE_bin_w'])
        FFT_full = np.complex64(np.squeeze(matread[fft_name]))
Somnath, Suhas's avatar
Somnath, Suhas committed
878
        # For whatever weird reason, the sign of the imaginary portion is flipped. Correct it:
Unknown's avatar
Unknown committed
879
        # BE_bin_FFT = np.conjugate(FFT_full[BE_bin_ind])
Somnath, Suhas's avatar
Somnath, Suhas committed
880
881
        BE_bin_FFT = np.zeros(len(BE_bin_ind), dtype=np.complex64)
        BE_bin_FFT.real = np.real(FFT_full[BE_bin_ind])
Unknown's avatar
Unknown committed
882
883
        BE_bin_FFT.imag = -1 * np.imag(FFT_full[BE_bin_ind])

Somnath, Suhas's avatar
Somnath, Suhas committed
884
        ex_wfm = np.real(np.fft.ifft(np.fft.ifftshift(FFT_full)))
Somnath, Suhas's avatar
Somnath, Suhas committed
885
886

        return BE_bin_ind, BE_bin_w, BE_bin_FFT, ex_wfm
Unknown's avatar
Unknown committed
887

Somnath, Suhas's avatar
Somnath, Suhas committed
888
    def __build_udvs_table(self, parm_dict):
Somnath, Suhas's avatar
Somnath, Suhas committed
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
        """
        Generates the UDVS table using the parameters
        
        Parameters 
        --------------------
        parm_dict : dictionary
            Parameters describing experiment
        
        Returns 
        --------------------      
        UD_VS_table_label : List of strings
            Labels for columns in the UDVS table
        UD_VS_table_unit : List of strings
            Units for the columns in the UDVS table
        UD_VS_table : 2D numpy float array
            UDVS data table
        """
Unknown's avatar
Unknown committed
906

Somnath, Suhas's avatar
Somnath, Suhas committed
907
        def translate_val(target, strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
908
909
            """
            Internal function - Interprets the provided value using the provided lookup table
Somnath, Suhas's avatar
Somnath, Suhas committed
910
911
912
913
914
915
916
917
918

            Parameters
            ----------
            target : String
                Item we are looking for in the strvals list
            strvals : list of strings
                List of source values
            numvals : list of numbers
                List of results
Somnath, Suhas's avatar
Somnath, Suhas committed
919
            """
Unknown's avatar
Unknown committed
920

Somnath, Suhas's avatar
Somnath, Suhas committed
921
            if len(strvals) is not len(numvals):
Unknown's avatar
Unknown committed
922
                return None
Somnath, Suhas's avatar
Somnath, Suhas committed
923
            for strval, fltval in zip(strvals, numvals):
Somnath, Suhas's avatar
Somnath, Suhas committed
924
925
                if target == strval:
                    return fltval
Somnath, Suhas's avatar
Somnath, Suhas committed
926
            return None  # not found in list
Unknown's avatar
Unknown committed
927
928

        # % Extract values from parm text file
Unknown's avatar
Unknown committed
929
        BE_signal_type = translate_val(parm_dict['BE_phase_content'],
Unknown's avatar
Unknown committed
930
931
932
                                       ['chirp-sinc hybrid', '1/2 harmonic excitation',
                                        '1/3 harmonic excitation', 'pure sine'],
                                       [1, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
933
934
935
936
937
938
        # This is necessary when normalzing the AI by the AO
        self.harmonic = BE_signal_type
        self.signal_type = BE_signal_type
        if BE_signal_type is 4:
            self.harmonic = 1
        BE_amp = parm_dict['BE_amplitude_[V]']
Unknown's avatar
Unknown committed
939

Somnath, Suhas's avatar
Somnath, Suhas committed
940
941
        VS_amp = parm_dict['VS_amplitude_[V]']
        VS_offset = parm_dict['VS_offset_[V]']
Unknown's avatar
Unknown committed
942
        # VS_read_voltage = parm_dict['VS_read_voltage_[V]']
Somnath, Suhas's avatar
Somnath, Suhas committed
943
944
        VS_steps = parm_dict['VS_steps_per_full_cycle']
        VS_cycles = parm_dict['VS_number_of_cycles']
Somnath, Suhas's avatar
Somnath, Suhas committed
945
946
947
        VS_fraction = translate_val(parm_dict['VS_cycle_fraction'],
                                    ['full', '1/2', '1/4', '3/4'],
                                    [1., 0.5, 0.25, 0.75])
Somnath, Suhas's avatar
Somnath, Suhas committed
948
949
        VS_shift = parm_dict['VS_cycle_phase_shift']
        if VS_shift is not 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
950
951
952
953
954
955
956
            VS_shift = translate_val(VS_shift, ['1/4', '1/2', '3/4'], [0.25, 0.5, 0.75])
        VS_in_out_cond = translate_val(parm_dict['VS_measure_in_field_loops'],
                                       ['out-of-field', 'in-field', 'in and out-of-field'], [0, 1, 2])
        VS_ACDC_cond = translate_val(parm_dict['VS_mode'],
                                     ['DC modulation mode', 'AC modulation mode with time reversal',
                                      'load user defined VS Wave from file', 'current mode'],
                                     [0, 2, 3, 4])
Somnath, Suhas's avatar
Somnath, Suhas committed
957
958
959
960
        self.expt_type = VS_ACDC_cond
        FORC_cycles = parm_dict['FORC_num_of_FORC_cycles']
        FORC_A1 = parm_dict['FORC_V_high1_[V]']
        FORC_A2 = parm_dict['FORC_V_high2_[V]']
Unknown's avatar
Unknown committed
961
        # FORC_repeats = parm_dict['# of FORC repeats']
Somnath, Suhas's avatar
Somnath, Suhas committed
962
963
        FORC_B1 = parm_dict['FORC_V_low1_[V]']
        FORC_B2 = parm_dict['FORC_V_low2_[V]']
Unknown's avatar
Unknown committed
964
965
966

        # % build vector of voltage spectroscopy values

Somnath, Suhas's avatar
Somnath, Suhas committed
967
        if VS_ACDC_cond == 0 or VS_ACDC_cond == 4:  # DC voltage spectroscopy or current mode
Unknown's avatar
Unknown committed
968
            VS_amp_vec_1 = np.arange(0, 1 + 1 / (VS_steps / 4), 1 / (VS_steps / 4))
Somnath, Suhas's avatar
Somnath, Suhas committed
969
970
            VS_amp_vec_2 = np.flipud(VS_amp_vec_1[:-1])
            VS_amp_vec_3 = -VS_amp_vec_1[1:]
Unknown's avatar
Unknown committed
971
972
            VS_amp_vec_4 = VS_amp_vec_1[1:-1] - 1
            vs_amp_vec = VS_amp * (np.hstack((VS_amp_vec_1, VS_amp_vec_2, VS_amp_vec_3, VS_amp_vec_4)))
Unknown's avatar
Unknown committed
973
            # apply phase shift to VS wave
Unknown's avatar
Unknown committed
974
            vs_amp_vec = np.roll(vs_amp_vec, int(np.floor(VS_steps / VS_fraction * VS_shift)))
Unknown's avatar
Unknown committed
975
            # cut VS waveform
Unknown's avatar
Unknown committed
976
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction))]
Unknown's avatar
Unknown committed
977
            # repeat VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
978
            vs_amp_vec = np.tile(vs_amp_vec, int(VS_cycles))
Unknown's avatar
Unknown committed
979
980
            vs_amp_vec = vs_amp_vec + VS_offset

Somnath, Suhas's avatar
Somnath, Suhas committed
981
        elif VS_ACDC_cond == 2:  # AC voltage spectroscopy with time reversal
Unknown's avatar
Unknown committed
982
983
            vs_amp_vec = VS_amp * np.arange(1 / (VS_steps / 2 / VS_fraction), 1 + 1 / (VS_steps / 2 / VS_fraction),
                                            1 / (VS_steps / 2 / VS_fraction))
Somnath, Suhas's avatar
Somnath, Suhas committed
984
            vs_amp_vec = np.roll(vs_amp_vec,
Unknown's avatar
Unknown committed
985
986
                                 int(np.floor(VS_steps / VS_fraction * VS_shift)))  # apply phase shift to VS wave
            vs_amp_vec = vs_amp_vec[:int(np.floor(VS_steps * VS_fraction / 2))]  # cut VS waveform
Somnath, Suhas's avatar
Somnath, Suhas committed
987
            vs_amp_vec = np.tile(vs_amp_vec, VS_cycles * 2)  # repeat VS waveform
Unknown's avatar
Unknown committed
988

Somnath, Suhas's avatar
Somnath, Suhas committed
989
        if FORC_cycles > 1:
Unknown's avatar
Unknown committed
990
991
992
993
994
995
996
            vs_amp_vec = vs_amp_vec / np.max(np.abs(vs_amp_vec))
            FORC_cycle_vec = np.arange(0, FORC_cycles + 1, FORC_cycles / (FORC_cycles - 1))
            FORC_A_vec = FORC_cycle_vec * (FORC_A2 - FORC_A1) / FORC_cycles + FORC_A1
            FORC_B_vec = FORC_cycle_vec * (FORC_B2 - FORC_B1) / FORC_cycles + FORC_B1
            FORC_amp_vec = (FORC_A_vec - FORC_B_vec) / 2
            FORC_off_vec = (FORC_A_vec + FORC_B_vec) / 2

997
            VS_amp_mat = np.tile(vs_amp_vec, [int(FORC_cycles), 1])
Somnath, Suhas's avatar
Somnath, Suhas committed
998
999
            FORC_amp_mat = np.tile(FORC_amp_vec, [len(vs_amp_vec), 1]).transpose()
            FORC_off_mat = np.tile(FORC_off_vec, [len(vs_amp_vec), 1]).transpose()
Unknown's avatar
Unknown committed
1000
            VS_amp_mat = VS_amp_mat * FORC_amp_mat + FORC_off_mat