beps_ndf.py 48.4 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
8
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:07:16 2015

@author: Suhas Somnath

"""

Somnath, Suhas's avatar
Somnath, Suhas committed
9
10
11
from __future__ import division  # int/int = float
import numpy as np
from os import path, listdir, remove
Somnath, Suhas's avatar
Somnath, Suhas committed
12
from warnings import warn
Somnath, Suhas's avatar
Somnath, Suhas committed
13
14
import xlrd as xlreader  # To read the UDVS spreadsheet
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15
from .utils import make_position_mat, generate_dummy_main_parms
Somnath, Suhas's avatar
Somnath, Suhas committed
16
17
18
19
from .be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, normalizeBEresponse, \
    createSpecVals
from ..microdata import MicroDataGroup, MicroDataset
from ..io_hdf5 import ioHDF5
Chris Smith's avatar
Chris Smith committed
20
21
from .translator import Translator
from ..be_hdf_utils import maxReadPixels
Chris Smith's avatar
Chris Smith committed
22
from ..hdf_utils import getH5DsetRefs, linkRefs, calc_chunks
Somnath, Suhas's avatar
Somnath, Suhas committed
23

24
25
26
27
nf32 = np.dtype([('super_band', np.float32), ('inter_bin_band', np.float32),
                 ('sub_band', np.float32)])


Somnath, Suhas's avatar
Somnath, Suhas committed
28
29
30
31
class BEPSndfTranslator(Translator):
    """
    Translates Band Excitation Polarization Switching (BEPS) datasets from .dat
    files to .h5
Chris Smith's avatar
Chris Smith committed
32

Somnath, Suhas's avatar
Somnath, Suhas committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    """       
        
    def translate(self, data_filepath, show_plots=True, save_plots=True, do_histogram=False, debug=False):
        """
        The main function that translates the provided file into a .h5 file
        
        Parameters
        ----------------
        data_filepath : String / unicode
            Absolute path of the data file (.dat)
        show_plots : Boolean (Optional. Default is True)
            Whether or not to show plots
        save_plots : Boolean (Optional. Default is True)
            Whether or not to save the generated plots
        do_histogram : Boolean (Optional. Default is False)
            Whether or not to generate and save 2D histograms of the raw data
        debug : Boolean (Optional. default is false)
            Whether or not to print log statements
            
        Returns
        --------------
        h5_path : String / unicode
            Absolute path of the generated .h5 file
Chris Smith's avatar
Chris Smith committed
56

Somnath, Suhas's avatar
Somnath, Suhas committed
57
58
        """
        ## Read the parameter files
59
60
61
        if debug:
            print('BEndfTranslator: Getting file paths')

62
        parm_filepath, udvs_filepath, parms_mat_path = self._parse_file_path(data_filepath)
63
64
65
66
        if debug:
            print('BEndfTranslator: Reading Parms text file')

        isBEPS, self.parm_dict = parmsToDict(parm_filepath)
Somnath, Suhas's avatar
Somnath, Suhas committed
67
68
69
70
71
72
73
74
75
76
77
        self.parm_dict['data_type'] = 'BEPSData'
        if not isBEPS:
            warn('This is NOT a BEPS new-data-format dataset!')
            return None
        
        """ Find out if this is a custom experiment and whether in and out of field were acquired
        For a standard experiment where only in / out field is acquired, zeros are stored
        even for those UDVS steps without band excitation"""
        self.field_mode = self.parm_dict['VS_measure_in_field_loops']
        expt_type = self.parm_dict['VS_mode']
        self.spec_label = getSpectroscopicParmLabel(expt_type)
78
        std_expt = expt_type in ['DC modulation mode', 'current mode']
Somnath, Suhas's avatar
Somnath, Suhas committed
79
80
81
82
83
84
85
86
        self.halve_udvs_steps = False
        ignored_plt_grps = []
        if std_expt and self.field_mode != 'in and out-of-field':
            self.halve_udvs_steps = True
            if self.field_mode == 'out-of-field':
                ignored_plt_grps = ['in-field']
            else:
                ignored_plt_grps = ['out-of-field']
87
88

        h5_path = path.join(self.folder_path, self.basename+'.h5')
Somnath, Suhas's avatar
Somnath, Suhas committed
89
90
91
        if path.exists(h5_path):
            remove(h5_path)
        
92
93
        if debug:
            print('BEndfTranslator: Preparing to read parms.mat file')
94
        self.BE_wave = self.__get_excit_wfm(parms_mat_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
95
        
96
97
        if debug:
            print('BEndfTranslator: About to read UDVS file')
Somnath, Suhas's avatar
Somnath, Suhas committed
98
        
99
        self.udvs_labs, self.udvs_units, self.udvs_mat = self.__read_udvs_table(udvs_filepath)
Somnath, Suhas's avatar
Somnath, Suhas committed
100
        # Remove the unused plot group columns before proceeding:
101
102
        self.udvs_mat, self.udvs_labs, self.udvs_units = trimUDVS(self.udvs_mat, self.udvs_labs, self.udvs_units,
                                                                  ignored_plt_grps)
Somnath, Suhas's avatar
Somnath, Suhas committed
103
104
        if debug: print('BEndfTranslator: Read UDVS file')
            
105
106
107
        self.num_udvs_steps = self.udvs_mat.shape[0]
        # This is absolutely crucial for reconstructing the data chronologically
        self.excit_type_vec = (self.udvs_mat[:, 4]).astype(int)
Somnath, Suhas's avatar
Somnath, Suhas committed
108
109
        
        # First figure out how many waveforms are present in the data from the UDVS
110
        unique_waves = self.__get_unique_wave_types(self.excit_type_vec) 
Somnath, Suhas's avatar
Somnath, Suhas committed
111
112
113
114
115
116
117
        self.__unique_waves__ = unique_waves
        self.__num_wave_types__ = len(unique_waves)
        # print self.__num_wave_types__, 'different excitation waveforms in this experiment'
        
        if debug: print('BEndfTranslator: Preparing to set up parsers')

        # Preparing objects to parse the file(s)
118
        parsers = self.__assemble_parsers()        
Somnath, Suhas's avatar
Somnath, Suhas committed
119
120
        
        # Gathering some basic details before parsing the files:
121
122
        self.max_pixels = parsers[0].get_num_pixels()
        s_pixels = np.array(parsers[0].get_spatial_pixels())
123
124
        self.pos_labels = ['Laser Spot', 'Z', 'X', 'Y']
        self.pos_labels = [self.pos_labels[i] for i in np.where(s_pixels > 1)[0]]
125
        self.pos_mat = make_position_mat(s_pixels)
126
        self.pos_units = ['um' for _ in range(len(self.pos_labels))]
Somnath, Suhas's avatar
Somnath, Suhas committed
127
128
129
#         self.pos_mat = np.int32(self.pos_mat)
        
        # Helping Eric out a bit. Remove this section at a later time:
130
        main_parms = generate_dummy_main_parms()
131
132
        main_parms['grid_size_x'] = self.parm_dict['grid_num_cols']
        main_parms['grid_size_y'] = self.parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
133
134
        main_parms['experiment_date'] = self.parm_dict['File_date_and_time']        
        # assuming that the experiment was completed:        
135
136
        main_parms['current_position_x'] = self.parm_dict['grid_num_cols']-1
        main_parms['current_position_y'] = self.parm_dict['grid_num_rows']-1
Somnath, Suhas's avatar
Somnath, Suhas committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
        main_parms['data_type'] = 'BEPSData'
        main_parms['translator'] = 'NDF'
        
        # Writing only the root now:
        spm_data = MicroDataGroup('')
        spm_data.attrs = main_parms
        self.hdf = ioHDF5(h5_path)
#         self.hdf.clear()
        
        #cacheSettings = self.hdf.file.id.get_access_plist().get_cache()
        #print 'H5 cache settings: Metadata Objects {}    Data Chunks {}    Raw Data Size(kB) {}'.format(cacheSettings[0],cacheSettings[1],cacheSettings[2]/1024)
        
        self.hdf.writeData(spm_data)
        
        ########################################################
        # Reading and parsing the .dat file(s) 
Chris Smith's avatar
Chris Smith committed
153
154

        self._read_data(parsers, unique_waves, show_plots=False, save_plots=True, do_histogram=False,)
Somnath, Suhas's avatar
Somnath, Suhas committed
155
        
Chris Smith's avatar
Chris Smith committed
156
157
158
        self.hdf.close()
            
        return h5_path
Somnath, Suhas's avatar
Somnath, Suhas committed
159

Chris Smith's avatar
Chris Smith committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
    def _read_data(self, parsers, unique_waves, show_plots, save_plots, do_histogram):
        """
        Loops over all pixels and reads the data into the HDF5 file.

        Parameters
        ----------
        parsers : list of BEPSndfPixel
            List of parser object that will read the pixel data from the files
        unique_waves : numpy.ndarray of int
            Array denoting the unique waveforms in the experiment
        show_plots : Boolean, optional
            Should generated plots be shown during translation.  Default True
        save_plots : Boolean, optional
            Should generated plots be saved to disk during translation.  Default True
        do_histogram : Boolean, optional
            Should histograms be generated for the different plot groups.  Default False

        Returns
        -------
        None
Chris Smith's avatar
Chris Smith committed
180

Chris Smith's avatar
Chris Smith committed
181
182
        """
        print('Reading data file(s)')
Somnath, Suhas's avatar
Somnath, Suhas committed
183
        self.dset_index = 0
Chris Smith's avatar
Chris Smith committed
184
        self.ds_pixel_start_indx = 0
Somnath, Suhas's avatar
Somnath, Suhas committed
185
        for pixel_ind in range(self.max_pixels):
Chris Smith's avatar
Chris Smith committed
186
187
188
189

            if (100.0 * (pixel_ind + 1) / self.max_pixels) % 10 == 0:
                print('{} % complete'.format(int(100 * (pixel_ind + 1) / self.max_pixels)))

Somnath, Suhas's avatar
Somnath, Suhas committed
190
191
192
            # First read the next pixel from all parsers:
            current_pixels = {}
            for prsr in parsers:
193
                current_pixels[prsr.get_wave_type()] = prsr.read_pixel()
Chris Smith's avatar
Chris Smith committed
194
195

            if pixel_ind == 0:
196
                h5_refs = self.__initialize_meas_group(self.max_pixels, current_pixels)
Chris Smith's avatar
Chris Smith committed
197
198
                prev_pixels = current_pixels  # This is here only to avoid annoying warnings.
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
199
                if current_pixels[unique_waves[0]].is_different_from(prev_pixels[unique_waves[0]]):
Somnath, Suhas's avatar
Somnath, Suhas committed
200
                    # Some parameter has changed. Write current group and make new group
201
                    self.__close_meas_group(h5_refs, show_plots, save_plots, do_histogram)
Somnath, Suhas's avatar
Somnath, Suhas committed
202
                    self.ds_pixel_start_indx = pixel_ind
203
                    h5_refs = self.__initialize_meas_group(self.max_pixels - pixel_ind, current_pixels)
Chris Smith's avatar
Chris Smith committed
204

Somnath, Suhas's avatar
Somnath, Suhas committed
205
            # print('reading Pixel {} of {}'.format(pixel_ind,self.max_pixels))
206
            self.__append_pixel_data(current_pixels)
Chris Smith's avatar
Chris Smith committed
207

Somnath, Suhas's avatar
Somnath, Suhas committed
208
            prev_pixels = current_pixels
209
        self.__close_meas_group(h5_refs, show_plots, save_plots, do_histogram)
Chris Smith's avatar
Chris Smith committed
210

Somnath, Suhas's avatar
Somnath, Suhas committed
211
212
    ###################################################################################################
        
213
    def __close_meas_group(self, h5_refs, show_plots, save_plots, do_histogram):
Somnath, Suhas's avatar
Somnath, Suhas committed
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
        """
        Performs following operations : 
            * Updates the number of pixels attribute in the measurement group
            * Writes Noise floor axis labels as region references
            * Writes position values and indices along with region references
            * Links all ancilliary datasets to the main data set
            * Writes the spatiall averaged plot data
        
        Parameters
        ----------
        h5_refs : list of HDF references
            References to the written datasets
        show_plots : Boolean 
            Whether or not to show plots
        save_plots : Boolean
            Whether or not to save the generated plots
        do_histogram : Boolean
            Whether or not to generate and save 2D histograms of the raw data
            
        Returns
        -------
        None
Chris Smith's avatar
Chris Smith committed
236

Somnath, Suhas's avatar
Somnath, Suhas committed
237
238
239
240
241
242
        """
        # Update the number of pixels in the attributes
        meas_grp = self.ds_main.parent
        meas_grp.attrs['num_pix'] = self.ds_pixel_index
        
        # Write position specific datasets now that the dataset is complete
243
        pos_slice_dict = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
244
        for spat_ind, spat_dim in enumerate(self.pos_labels):
245
246
247
            pos_slice_dict[spat_dim] = (slice(None), slice(spat_ind, spat_ind+1))
        ds_pos_ind = MicroDataset('Position_Indices', self.pos_mat[self.ds_pixel_start_indx:
                                  self.ds_pixel_start_indx + self.ds_pixel_index, :], dtype=np.uint)
Somnath, Suhas's avatar
Somnath, Suhas committed
248
249
        ds_pos_ind.attrs['labels'] = pos_slice_dict
        ds_pos_ind.attrs['units'] = self.pos_units
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278

        self.pos_vals_list = np.array(self.pos_vals_list)
        # Ensuring that the X and Y values vary from 0 to N instead of -0.5 N to + 0.5 N
        for col_ind in range(2):
            min_val = np.min(self.pos_vals_list[:, col_ind])
            self.pos_vals_list[:, col_ind] -= min_val
            self.pos_vals_list[:, col_ind] *= 1E+6  # convert to microns

        if np.max(self.pos_vals_list[:, 2]) > 1E-3:
            # Setpoint spectroscopy
            if 'Z' in self.pos_labels:
                dim_ind = self.pos_labels.index('Z')
                # TODO: Find a way to correct the labels
                # self.pos_labels[dim_ind] = 'Setpoint'
                self.pos_units[dim_ind] = 'defl V'
        else:
            # Z spectroscopy
            self.pos_vals_list[:, 2] *= 1E+6  # convert to microns

        pos_val_mat = np.float32(self.pos_mat[self.ds_pixel_start_indx:
                                              self.ds_pixel_start_indx + self.ds_pixel_index, :])

        for col_ind, targ_dim_name in enumerate(['X', 'Y', 'Z']):
            if targ_dim_name in self.pos_labels:
                dim_ind = self.pos_labels.index(targ_dim_name)
                # Replace indices with the x, y, z values from the pixels
                pos_val_mat[:, dim_ind] = self.pos_vals_list[:, col_ind]

        ds_pos_val = MicroDataset('Position_Values', pos_val_mat)
Somnath, Suhas's avatar
Somnath, Suhas committed
279
280
        ds_pos_val.attrs['labels'] = pos_slice_dict
        ds_pos_val.attrs['units'] = self.pos_units
281

282
        meas_grp = MicroDataGroup(meas_grp.name, '/')
Somnath, Suhas's avatar
Somnath, Suhas committed
283
284
285
286
287
        meas_grp.addChildren([ds_pos_ind, ds_pos_val])
        
        h5_refs += self.hdf.writeData(meas_grp)
        
        # Do all the reference linking:
288
289
290
        aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values', 'UDVS_Indices',
                        'Spectroscopic_Indices', 'Bin_Step', 'Bin_Indices', 'Bin_Wfm_Type',
                        'Bin_Frequencies', 'Bin_FFT', 'UDVS', 'UDVS_Labels', 'Noise_Floor', 'Spectroscopic_Values']
291
        linkRefs(self.ds_main, getH5DsetRefs(aux_ds_names, h5_refs))
Somnath, Suhas's avatar
Somnath, Suhas committed
292
293
294
295
296
297

        # While we have all the references and mean data, write the plot groups as well:
        generatePlotGroups(self.ds_main, self.hdf, self.mean_resp, 
                           self.folder_path, self.basename,
                           self.max_resp, self.min_resp, 
                           max_mem_mb=self.max_ram,
Chris Smith's avatar
Chris Smith committed
298
                           spec_label=self.spec_label,
Somnath, Suhas's avatar
Somnath, Suhas committed
299
300
301
302
303
304
                           show_plots=show_plots, save_plots=save_plots,
                           do_histogram=do_histogram)
        
        # Now that everything about this dataset is complete:
        self.dset_index += 1
        
305
    # ##################################################################################################
Somnath, Suhas's avatar
Somnath, Suhas committed
306
            
307
    def __initialize_meas_group(self, num_pix, current_pixels):
Somnath, Suhas's avatar
Somnath, Suhas committed
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
        """
        Creates and initializes the primary (and auxillary) datasets and datagroups
        to hold the raw data for the current set of experimental parameters.
        
        Parameters
        ----------
        num_pix : unsigned int
            Number of pixels this datagroup is expected to hold
        current_pixels : dictionary of BEPSndfPixel objects
            Extracted data for the first pixel in this group
            
        Returns
        ---------
        h5_refs : list of HDF5group and HDF5Dataset references 
            references of the written H5 datasets
Chris Smith's avatar
Chris Smith committed
323

Somnath, Suhas's avatar
Somnath, Suhas committed
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        """

        tot_bins = 0
        tot_pts = 0
        # Each wavetype can have different number of bins
        for pixl in current_pixels.values():
            tot_bins += pixl.num_bins
            tot_pts += pixl.num_bins*pixl.num_steps
            
        # Need to halve the number of steps when only in / out field is acquired:
        if self.halve_udvs_steps:
            tot_pts = int(tot_pts/2)
        
        # Populate information from the columns within the pixels such as the FFT, bin freq, indices, etc. 
338
339
340
341
342
        bin_freqs = np.zeros(shape=tot_bins, dtype=np.float32)
        bin_inds = np.zeros(shape=tot_bins, dtype=np.uint32)
        bin_FFT = np.zeros(shape=tot_bins, dtype=np.complex64)
        exec_bin_vec = np.zeros(shape=tot_bins, dtype=np.int32)
        pixel_bins = {}  # Might be useful later
Somnath, Suhas's avatar
Somnath, Suhas committed
343
344
345
346
347
348
349
        stind = 0
        for wave_type in self.__unique_waves__:
            pixl = current_pixels[wave_type]
            exec_bin_vec[stind:stind+pixl.num_bins] = wave_type*np.ones(pixl.num_bins)
            bin_inds[stind:stind+pixl.num_bins] = pixl.BE_bin_ind
            bin_freqs[stind:stind+pixl.num_bins] = pixl.BE_bin_w
            bin_FFT[stind:stind+pixl.num_bins] = pixl.FFT_BE_wave
350
351
            pixel_bins[wave_type] = [stind, pixl.num_bins]
            stind += pixl.num_bins
Somnath, Suhas's avatar
Somnath, Suhas committed
352
353
354
        del pixl, stind 
                
        # Make the index matrix that has the UDVS step number and bin indices
355
        spec_inds = np.zeros(shape=(2, tot_pts), dtype=np.uint32)
Somnath, Suhas's avatar
Somnath, Suhas committed
356
357
358
        stind = 0
        # Need to go through the UDVS file and reconstruct chronologically
        for step_index, wave_type in enumerate(self.excit_type_vec):
359
360
            if self.halve_udvs_steps and self.udvs_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
                    continue  # skip
Somnath, Suhas's avatar
Somnath, Suhas committed
361
            vals = pixel_bins[wave_type]
362
363
            spec_inds[1, stind:stind+vals[1]] = step_index * np.ones(vals[1])  # UDVS step
            spec_inds[0, stind:stind+vals[1]] = np.arange(vals[0], vals[0]+vals[1])  # Bin step
Somnath, Suhas's avatar
Somnath, Suhas committed
364
            stind += vals[1]
365
        del stind, wave_type, step_index
Somnath, Suhas's avatar
Somnath, Suhas committed
366
        
367
        self.spec_inds = spec_inds  # will need this for plot group generation
Somnath, Suhas's avatar
Somnath, Suhas committed
368
369
370
                        
        ds_ex_wfm = MicroDataset('Excitation_Waveform', self.BE_wave)
        ds_bin_freq = MicroDataset('Bin_Frequencies', bin_freqs)
371
        ds_bin_inds = MicroDataset('Bin_Indices', bin_inds - 1, dtype=np.uint32)  # From Matlab to Python (base 0)
372
        ds_bin_fft = MicroDataset('Bin_FFT', bin_FFT)
Somnath, Suhas's avatar
Somnath, Suhas committed
373
374
375
376
377
378
379
380
381
382
        ds_wfm_typ = MicroDataset('Bin_Wfm_Type', exec_bin_vec)
        ds_bin_steps = MicroDataset('Bin_Step', np.arange(tot_bins, dtype=np.uint32)) 
        
        curr_parm_dict = self.parm_dict
        # Some very basic information that can help the processing crew
        curr_parm_dict['num_bins'] = tot_bins
        curr_parm_dict['num_pix'] = num_pix
                
        # technically should change the date, etc.
        self.current_group = '{:s}'.format('Measurement_')
383
        meas_grp = MicroDataGroup(self.current_group, '/')
Somnath, Suhas's avatar
Somnath, Suhas committed
384
385
386
387
388
389
        meas_grp.attrs = curr_parm_dict

        chan_grp = MicroDataGroup('Channel_')
        chan_grp.attrs['Channel_Input'] = curr_parm_dict['IO_Analog_Input_1']
        meas_grp.addChildren([chan_grp])
        
390
        udvs_slices = dict()
Somnath, Suhas's avatar
Somnath, Suhas committed
391
        for col_ind, col_name in enumerate(self.udvs_labs):
392
393
394
395
396
            udvs_slices[col_name] = (slice(None), slice(col_ind, col_ind+1))
            # print('UDVS column index {} = {}'.format(col_ind,col_name))
        ds_udvs_mat = MicroDataset('UDVS', self.udvs_mat)
        ds_udvs_mat.attrs['labels'] = udvs_slices
        ds_udvs_mat.attrs['units'] = self.udvs_units
Somnath, Suhas's avatar
Somnath, Suhas committed
397
398
399
        
        actual_udvs_steps = self.num_udvs_steps
        if self.halve_udvs_steps:
400
401
402
403
            actual_udvs_steps /= 2
        if actual_udvs_steps % 1:
            raise ValueError('Actual number of UDVS steps should be an integer')
        actual_udvs_steps = int(actual_udvs_steps)
Somnath, Suhas's avatar
Somnath, Suhas committed
404
405
406
        
        curr_parm_dict['num_udvs_steps'] = actual_udvs_steps        
        
407
408
409
        ds_udvs_inds = MicroDataset('UDVS_Indices', self.spec_inds[1])
        # ds_udvs_inds.attrs['labels'] = {'UDVS_step':(slice(None),)}

Chris Smith's avatar
Chris Smith committed
410
        '''
Somnath, Suhas's avatar
Somnath, Suhas committed
411
        Create the Spectroscopic Values tables
Chris Smith's avatar
Chris Smith committed
412
        '''
Chris Smith's avatar
Chris Smith committed
413
414
415
416
        spec_vals, spec_inds, spec_vals_labs, spec_vals_units, spec_vals_labs_names = \
            createSpecVals(self.udvs_mat, spec_inds, bin_freqs, exec_bin_vec,
                           curr_parm_dict, np.array(self.udvs_labs), self.udvs_units)

Somnath, Suhas's avatar
Somnath, Suhas committed
417
418
        spec_vals_slices = dict()
        for row_ind, row_name in enumerate(spec_vals_labs):
419
420
            spec_vals_slices[row_name] = (slice(row_ind, row_ind+1), slice(None))
        ds_spec_vals_mat = MicroDataset('Spectroscopic_Values', np.array(spec_vals, dtype=np.float32))
Somnath, Suhas's avatar
Somnath, Suhas committed
421
422
423
424
425
426
        ds_spec_vals_mat.attrs['labels'] = spec_vals_slices
        ds_spec_vals_mat.attrs['units'] = spec_vals_units
        ds_spec_mat = MicroDataset('Spectroscopic_Indices', spec_inds, dtype=np.uint32)
        ds_spec_mat.attrs['labels'] = spec_vals_slices
        ds_spec_mat.attrs['units'] = spec_vals_units  
        for entry in spec_vals_labs_names:
427
            label = entry[0]+'_parameters'
Somnath, Suhas's avatar
Somnath, Suhas committed
428
            names = entry[1]
429
430
            ds_spec_mat.attrs[label] = names
            ds_spec_vals_mat.attrs[label] = names
Somnath, Suhas's avatar
Somnath, Suhas committed
431

Chris Smith's avatar
Chris Smith committed
432
        '''
Somnath, Suhas's avatar
Somnath, Suhas committed
433
434
435
436
437
        New Method for chunking the Main_Data dataset.  Chunking is now done in N-by-N squares of UDVS steps by pixels.
        N is determined dinamically based on the dimensions of the dataset.  Currently it is set such that individual
        chunks are less than 10kB in size.
        
        Chris Smith -- csmith55@utk.edu
Chris Smith's avatar
Chris Smith committed
438
        '''
Somnath, Suhas's avatar
Somnath, Suhas committed
439
        max_bins_per_pixel = np.max(pixel_bins.values())
Chris Smith's avatar
Chris Smith committed
440

Chris Smith's avatar
Chris Smith committed
441
442
443
444
445
446
447
448
        beps_chunks = calc_chunks([num_pix, tot_pts],
                                  np.complex64(0).itemsize,
                                  unit_chunks=(1, max_bins_per_pixel))
        ds_main_data = MicroDataset('Raw_Data',
                                    np.zeros(shape=(1, tot_pts), dtype=np.complex64),
                                    chunking=beps_chunks,
                                    resizable=True,
                                    compression='gzip')
449
450
451

        ds_noise = MicroDataset('Noise_Floor', np.zeros(shape=(1, actual_udvs_steps), dtype=nf32),
                                chunking=(1, actual_udvs_steps), resizable=True, compression='gzip')
452

Somnath, Suhas's avatar
Somnath, Suhas committed
453
454
455
456
        # Allocate space for the first pixel for now and write along with the complete tree...
        # Positions CANNOT be written at this time since we don't know if the parameter changed
        
        chan_grp.addChildren([ds_main_data, ds_noise, ds_ex_wfm, ds_spec_mat, ds_wfm_typ,
457
458
                              ds_bin_steps, ds_bin_inds, ds_bin_freq, ds_bin_fft, ds_udvs_mat,
                              ds_spec_vals_mat, ds_udvs_inds])
Somnath, Suhas's avatar
Somnath, Suhas committed
459
                              
460
        # meas_grp.showTree()
Somnath, Suhas's avatar
Somnath, Suhas committed
461
462
463
464
        h5_refs = self.hdf.writeData(meas_grp)
        
        self.ds_noise = getH5DsetRefs(['Noise_Floor'], h5_refs)[0] 
        self.ds_main = getH5DsetRefs(['Raw_Data'], h5_refs)[0]
465
        self.pos_vals_list = list()
Somnath, Suhas's avatar
Somnath, Suhas committed
466
                
467
        # self.dset_index += 1 #  raise dset index after closing only
Somnath, Suhas's avatar
Somnath, Suhas committed
468
469
470
        self.ds_pixel_index = 0
        
        # Use this for plot groups:
471
        self.mean_resp = np.zeros(shape=tot_pts, dtype=np.complex64)
Somnath, Suhas's avatar
Somnath, Suhas committed
472
473
        
        # Used for Histograms
474
475
        self.max_resp = np.zeros(shape=num_pix, dtype=np.float32)
        self.min_resp = np.zeros(shape=num_pix, dtype=np.float32)
Somnath, Suhas's avatar
Somnath, Suhas committed
476
477
478
        
        return h5_refs
        
479
    # ##################################################################################################
Somnath, Suhas's avatar
Somnath, Suhas committed
480
        
481
    def __append_pixel_data(self, pixel_data):
Somnath, Suhas's avatar
Somnath, Suhas committed
482
483
484
485
486
487
488
489
490
491
492
493
        """
        Goes through the list of pixel objects and populates the raw dataset 
        and noise dataset for this spatial pixel.
        
        Parameters
        ----------
        pixel_data : List of BEPSndfPixel objects 
            List containing parsed data for this particular spatial pixel
        
        Returns
        ---------
        None
Chris Smith's avatar
Chris Smith committed
494

Somnath, Suhas's avatar
Somnath, Suhas committed
495
496
497
498
499
500
501
502
        """
        
        if self.__num_wave_types__ == 1 and not self.halve_udvs_steps:
            """Technically, this will be taken care of in the later (general) part but 
            since this condition is more common it is worth writing for specifically"""
            
            data_vec = pixel_data[self.__unique_waves__[0]].spectrogram_vec
            noise_mat = np.float32(pixel_data[self.__unique_waves__[0]].noise_floor_mat)
503
504
505
506

            # Storing a list of lists since we don't know how many pixels we will find in this measurement group
            self.pos_vals_list.append([pixel_data[0].x_value, pixel_data[0].y_value,
                                       pixel_data[0].z_value])
Somnath, Suhas's avatar
Somnath, Suhas committed
507
508
509
510
            
        else:

            data_vec = np.zeros(shape=(self.ds_main.shape[1]), dtype=np.complex64)
511
            noise_mat = np.zeros(shape=(3, self.ds_noise.shape[1]), dtype=np.float32)
Somnath, Suhas's avatar
Somnath, Suhas committed
512
513
514
515
516
517
518
519
520
521
522
            
            internal_step_index = {}
            for wave_type in self.__unique_waves__:
                internal_step_index[wave_type] = 0
                
            stind = 0
            step_counter = 0    
            # Go through each line in the UDVS file and reconstruct chronologically
            for step_index, wave_type in enumerate(self.excit_type_vec):
                # get the noise and data from correct pixel -> address by wave_number.
                
523
                if self.halve_udvs_steps and self.udvs_mat[step_index, 2] < 1E-3:  # invalid AC amplitude
Somnath, Suhas's avatar
Somnath, Suhas committed
524
                    # print('Step index {} was skipped'.format(step_index))
525
                    # Not sure why each wave type has its own counter but there must have been a good reason
Somnath, Suhas's avatar
Somnath, Suhas committed
526
                    internal_step_index[wave_type] += 1 
527
                    continue  # skip
Somnath, Suhas's avatar
Somnath, Suhas committed
528
529
530
531
532
                
                data_pix = pixel_data[wave_type].spectrogram_mat
                noise_pix = pixel_data[wave_type].noise_floor_mat
                enind = stind + pixel_data[wave_type].num_bins
                                
533
534
                data_vec[stind:enind] = data_pix[:, internal_step_index[wave_type]]
                noise_mat[:, step_counter] = np.float32(noise_pix[:, internal_step_index[wave_type]])
Somnath, Suhas's avatar
Somnath, Suhas committed
535
                
536
                stind = enind
Somnath, Suhas's avatar
Somnath, Suhas committed
537
538
                internal_step_index[wave_type] += 1
                step_counter += 1
539
540
541
542

            # Storing a list of lists since we don't know how many pixels we will find in this measurement group
            self.pos_vals_list.append([pixel_data[wave_type].x_value, pixel_data[wave_type].y_value,
                                       pixel_data[wave_type].z_value])
Somnath, Suhas's avatar
Somnath, Suhas committed
543
544
545
546
547
548
            
            del internal_step_index, stind, enind, step_index, wave_type, step_counter
        
        if self.ds_pixel_index > 0:
            # in the case of the first pixel, we already reserved zeros- no extension
            # for all other lines - we extend the dataset before writing
549
550
            self.ds_main.resize(self.ds_main.shape[0]+1, axis=0)
            self.ds_noise.resize(self.ds_noise.shape[0]+1, axis=0)
Somnath, Suhas's avatar
Somnath, Suhas committed
551

552
553
554
        self.ds_main[-1, :] = data_vec
        self.ds_noise[-1] = np.array([tuple(noise) for noise in noise_mat.T], dtype=nf32)

Somnath, Suhas's avatar
Somnath, Suhas committed
555
556
557
558
559
560
561
562
563
564
565
566
        self.hdf.file.flush()
        
        # Take mean response here:
        self.mean_resp = (1/(self.ds_pixel_index + 1))*(data_vec + self.ds_pixel_index*self.mean_resp)
        
        self.max_resp[self.ds_pixel_index] = np.amax(np.abs(data_vec))
        self.min_resp[self.ds_pixel_index] = np.amin(np.abs(data_vec))
        
        self.ds_pixel_index += 1
               
    ###################################################################################################
    
567
    def _parse_file_path(self, file_path):
Somnath, Suhas's avatar
Somnath, Suhas committed
568
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
569
        Returns the file paths to the parms text file and UDVS spreadsheet.\n
Somnath, Suhas's avatar
Somnath, Suhas committed
570
571
572
        Note: This function also initializes the basename and the folder_path for this instance
        
        Parameters
Chris Smith's avatar
Chris Smith committed
573
        ----------
574
575
        file_path : String / unicode
            Absolute path of the any file inside the measurment folder
Somnath, Suhas's avatar
Somnath, Suhas committed
576
577
        
        Returns
Chris Smith's avatar
Chris Smith committed
578
        -------
Somnath, Suhas's avatar
Somnath, Suhas committed
579
580
581
582
583
584
        parm_filepath : String / unicode
            absolute filepath of the parms text file
        udvs_filepath : String / unicode
            absolute file path of the UDVS spreadsheet
        parms_mat_path : String / unicode
            absolute filepath of the .mat parms file
Chris Smith's avatar
Chris Smith committed
585

Somnath, Suhas's avatar
Somnath, Suhas committed
586
        """
587
        udvs_filepath = None
588
589
590
591
592
593
594
595
596
597
598
599
        folder_path, tail = path.split(file_path)
        main_folder_path = None
        for item in listdir(folder_path):
            if item == 'newdataformat':
                main_folder_path = folder_path
                self.folder_path = path.join(folder_path, item)
                break
        if main_folder_path is None:
            self.folder_path = folder_path
            # need to set the parent as the root_folder
            main_folder_path, tail = path.split(folder_path)

Somnath, Suhas's avatar
Somnath, Suhas committed
600
        parms_mat_path = None
Chris Smith's avatar
Chris Smith committed
601
        parm_filepath = None
Somnath, Suhas's avatar
Somnath, Suhas committed
602
603
        for filenames in listdir(main_folder_path):
            if filenames.endswith('.txt') and filenames.find('parm') > 0:
Chris Smith's avatar
Chris Smith committed
604
                parm_filepath = path.join(main_folder_path, filenames)
Somnath, Suhas's avatar
Somnath, Suhas committed
605
            elif filenames.endswith('more_parms.mat'):
Chris Smith's avatar
Chris Smith committed
606
                parms_mat_path = path.join(main_folder_path, filenames)
Somnath, Suhas's avatar
Somnath, Suhas committed
607
608
        for filenames in listdir(self.folder_path):
            if (filenames.endswith('.xlsx') or filenames.endswith('.xls')) and filenames.find('UD_VS') > 0:
Chris Smith's avatar
Chris Smith committed
609
                udvs_filepath = path.join(self.folder_path, filenames)
Somnath, Suhas's avatar
Somnath, Suhas committed
610
                break
611
612
613
614
        if parm_filepath is None:
            for filenames in listdir(self.folder_path):
                if (filenames.endswith('.xls') or filenames.endswith('.xlsx')) and filenames.find('parm') > 0:
                    parm_filepath = path.join(main_folder_path, filenames)
Somnath, Suhas's avatar
Somnath, Suhas committed
615
                
Chris Smith's avatar
Chris Smith committed
616
        (tail, self.basename) = path.split(main_folder_path)
Somnath, Suhas's avatar
Somnath, Suhas committed
617
        
Chris Smith's avatar
Chris Smith committed
618
        return parm_filepath, udvs_filepath, parms_mat_path
Somnath, Suhas's avatar
Somnath, Suhas committed
619
620
        
    ###################################################################################################
Chris Smith's avatar
Chris Smith committed
621

622
    def __assemble_parsers(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
623
        """
624
        Returns a list of BEPSndfParser objects per excitation wave type
Somnath, Suhas's avatar
Somnath, Suhas committed
625
626
627
628
629
630
        
        Returns
        ---------
        parsers: list of BEPSndfParser objects 
            list of the same length as the input numpy array
        """
631
        parsers = []  # Maybe this needs to be a dictionary instead for easier access?
Somnath, Suhas's avatar
Somnath, Suhas committed
632
633
634
635
636
637
        for wave_type in self.__unique_waves__:
            filename = self.basename + '_1_'
            if wave_type > 0:
                filename = self.basename + '_1_' + str(wave_type) + '.dat'
            else:
                filename = self.basename + '_1_r' + str(abs(wave_type)) + '.dat'
638
            datapath = path.join(self.folder_path, filename)
639
640
641
            if not path.isfile(datapath):
                raise LookupError('Error!!: {}expected but not found!'.format(filename))
                # return
642
            parsers.append(BEPSndfParser(datapath, wave_type))
Somnath, Suhas's avatar
Somnath, Suhas committed
643
644
        return parsers
        
645
    # ##################################################################################################
Somnath, Suhas's avatar
Somnath, Suhas committed
646
        
647
648
    @staticmethod
    def __get_excit_wfm(filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
649
650
651
652
653
654
655
656
657
658
659
660
        """
        Returns the excitation BE waveform present in the more parms.mat file
        
        Parameters
        ------------
        filepath : String / unicode
            Absolute filepath of the .mat parameter file
        
        Returns
        -----------
        ex_wfm : 1D numpy float array
            Band Excitation waveform
Chris Smith's avatar
Chris Smith committed
661

Somnath, Suhas's avatar
Somnath, Suhas committed
662
663
664
665
666
        """
        if not path.exists(filepath):
            warn('BEPSndfTranslator - NO more_parms.mat file found')
            return np.zeros(1000, dtype=np.float32)
            
667
668
669
        matread = loadmat(filepath, variable_names=['FFT_BE_wave'])
        fft_full = np.complex64(np.squeeze(matread['FFT_BE_wave']))
        return np.float32(np.real(np.fft.ifft(np.fft.ifftshift(fft_full))))
Somnath, Suhas's avatar
Somnath, Suhas committed
670
        
671
    # ##################################################################################################
Somnath, Suhas's avatar
Somnath, Suhas committed
672
        
673
674
    @staticmethod
    def __read_udvs_table(udvs_filepath):
Somnath, Suhas's avatar
Somnath, Suhas committed
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
        """
        Reads the UDVS spreadsheet in either .xls or .xlsx format!
        
        Parameters
        ----------
        udvs_filepath : String / unicode
            absolute path to the spreadsheet
            
        Returns
        ----------
        udvs_labs : list of strings
            names of columns in the UDVS table
        udvs_units : list of strings
            units for columns in the UDVS table
        UDVS_mat : 2D numpy float array
            Contents of the UDVS table
Chris Smith's avatar
Chris Smith committed
691

Somnath, Suhas's avatar
Somnath, Suhas committed
692
693
694
695
        """
        workbook = xlreader.open_workbook(udvs_filepath)
        worksheet = workbook.sheet_by_index(0)
        udvs_labs = list()
Somnath, Suhas's avatar
Somnath, Suhas committed
696
        for col in range(worksheet.ncols):
697
            udvs_labs.append(str(worksheet.cell(0, col).value))
Somnath, Suhas's avatar
Somnath, Suhas committed
698
        # sometimes, the first few columns are named incorrectly. FORCE them to be named correclty:
Somnath, Suhas's avatar
Somnath, Suhas committed
699
        udvs_units = list(['' for _ in range(len(udvs_labs))])
700
        udvs_labs[0:5] = ['step_num', 'dc_offset', 'ac_amp', 'wave_type', 'wave_mod']
Somnath, Suhas's avatar
Somnath, Suhas committed
701
702
        udvs_units[0:5] = ['', 'V', 'A', '', '']
        
703
704
        udvs_mat = np.zeros(shape=(worksheet.nrows-1, worksheet.ncols), dtype=np.float32)
        for row in range(1, worksheet.nrows):
Somnath, Suhas's avatar
Somnath, Suhas committed
705
            for col in range(worksheet.ncols):
Somnath, Suhas's avatar
Somnath, Suhas committed
706
                try:
707
                    udvs_mat[row-1, col] = worksheet.cell(row, col).value
Somnath, Suhas's avatar
Somnath, Suhas committed
708
                except ValueError:
709
                    udvs_mat[row-1, col] = float('NaN')
Somnath, Suhas's avatar
Somnath, Suhas committed
710
711
712
713
                except:
                    raise
                    
        # Decrease counter of number of steps by 1 (Python base 0)
714
        udvs_mat[:, 0] -= 1
Somnath, Suhas's avatar
Somnath, Suhas committed
715
        
716
        return udvs_labs, udvs_units, udvs_mat
Somnath, Suhas's avatar
Somnath, Suhas committed
717

718
719
    @staticmethod
    def __get_unique_wave_types(vec):
Somnath, Suhas's avatar
Somnath, Suhas committed
720
721
722
723
724
725
726
727
728
729
730
731
732
        """
        Returns a numpy array containing the different waveforms in a 
        BEPS experiment in the format: [1,-1,2,-2.....]
        
        Parameters
        ---------------
        vec : 1D list or 1D numpy array 
            waveform types in the UDVS table
            
        Returns
        ----------
        uniq : 1D numpy array
            Unique waveform types in format listed above
Chris Smith's avatar
Chris Smith committed
733

Somnath, Suhas's avatar
Somnath, Suhas committed
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
        """
        sorted_all = np.unique(vec)
        pos_vals = sorted_all[sorted_all >= 0]
        neg_vals = sorted_all[sorted_all < 0]
        
        if len(pos_vals) == 0:
            return neg_vals
            
        if len(neg_vals) == 0:
            return pos_vals
            
        uniq = []
        posind = 0 
        negind = len(neg_vals)-1
        while posind < len(pos_vals) or negind >= 0:
            
            if posind == len(pos_vals):
751
                uniq += list(neg_vals)
Somnath, Suhas's avatar
Somnath, Suhas committed
752
753
                break
            if negind == len(neg_vals):
754
                uniq += list(pos_vals)
Somnath, Suhas's avatar
Somnath, Suhas committed
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
                break
            # Otherwise compare
            if pos_vals[posind] < abs(neg_vals[negind]):
                uniq.append(pos_vals[posind])
                posind += 1
            elif pos_vals[posind] == abs(neg_vals[negind]):
                uniq.append(pos_vals[posind])
                uniq.append(neg_vals[negind])
                posind += 1
                negind -= 1
            else:
                uniq.append(neg_vals[negind])
                negind -= 1
                        
        return np.array(uniq)
770

Somnath, Suhas's avatar
Somnath, Suhas committed
771
772
773
774
775
776
777
778
779
780

class BEPSndfParser(object):
    """
    An object of this class is given the responsibility to step through a 
    BEPS new data format file and return parsed BEPSndfPixel objects.\n
    This class is NOT responsible for actually parsing the byte contents within
    each pixel.\n
    Each wave type is given its own Parser object since it has a file of its own
    """
    
Somnath, Suhas's avatar
Somnath, Suhas committed
781
    def __init__(self, file_path, wave_type=1, scout=True):
Somnath, Suhas's avatar
Somnath, Suhas committed
782
783
784
785
786
787
788
789
790
791
792
793
        """
        Initializes the BEPSndfParser object with following inputs:
        
        Parameters
        -----------
        file_path : string or unicode
            Absolute path of the .dat file
        wave_type : int (optional. Default = 1)
            Integer value signifying type of the excitation waveform\n
        scout : Boolean (optional. Default = true) 
            whether or not the parser should figure out basic details such as 
            the number of pixels, and the spatial dimensionality
Chris Smith's avatar
Chris Smith committed
794

Somnath, Suhas's avatar
Somnath, Suhas committed
795
796
797
798
799
800
801
802
803
804
805
        """
        self.__file_handle__ = open(file_path, "rb")
        self.__EOF__ = False
        self.__curr_Pixel__ = 0
        self.__start_point__ = 0
        self.__wave_type__ = wave_type
        self.__filesize__ = path.getsize(file_path)
        self.__pixel_indices__ = list()
        if scout:
            self.__scout()
        
806
    def get_wave_type(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
807
808
809
810
        """
        Returns the excitation wave type as an integer
        
        Returns
811
        -------
Somnath, Suhas's avatar
Somnath, Suhas committed
812
813
        wave_type : int
            Wave type. Positive number means chirp up, negative number is chirp down.
Chris Smith's avatar
Chris Smith committed
814

Somnath, Suhas's avatar
Somnath, Suhas committed
815
816
817
        """
        return self.__wave_type__
        
818
    def get_num_pixels(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
819
820
821
822
        """
        Returns the total number of spatial pixels. This includes X, Y, Z, Laser positions
        
        Returns
823
        -------
Somnath, Suhas's avatar
Somnath, Suhas committed
824
825
        num_pix : unsigned int
            Number of pixels in this file
Chris Smith's avatar
Chris Smith committed
826

Somnath, Suhas's avatar
Somnath, Suhas committed
827
828
829
        """
        return self.__num_pixels__
        
830
    def get_spatial_pixels(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
831
832
833
834
835
        """
        Returns the number of steps in each spatial dimension 
        organized from fastest to slowest varying dimension
        
        Returns
836
        -------
837
838
839
840
841
842
843
844
        __num_laser_steps__ : unsigned int
            Number of laser steps
        __num_z_steps__ : unsigned int
            Number of height steps
        __num_x_steps__ : unsigned int
            Number of columns
        __num_y_steps__ : unsigned int
            Number of rows
Somnath, Suhas's avatar
Somnath, Suhas committed
845
        """
846
        return self.__num_laser_steps__, self.__num_z_steps__, self.__num_x_steps__, self.__num_y_steps__
Somnath, Suhas's avatar
Somnath, Suhas committed
847
848
849
850
851
852
853
854
855
856
    
    # Don't use this to figure out if something changes. You need pixel to previous pixel comparison    
    def __scout(self):
        """
        Steps through the file quickly without parsing it. 
        The idea is to calculate the number of pixels ahead of time so that 
        it is easier to parse the dataset. 
        For phase checking, it is recommended that this function be modified to 
        also keep track of the byte positions of the pixels so that pixels can be 
        directly accessed if need be.
Chris Smith's avatar
Chris Smith committed
857

Somnath, Suhas's avatar
Somnath, Suhas committed
858
859
860
861
862
        """
        count = 0
        self.__num_pixels__ = 0
        while True:
            self.__pixel_indices__.append(self.__start_point__*4)
Somnath, Suhas's avatar
Somnath, Suhas committed
863
864
            self.__file_handle__.seek(self.__start_point__*4, 0)
            spectrogram_length = int(np.fromstring(self.__file_handle__.read(4), dtype='f')[0])  # length of spectrogram
Somnath, Suhas's avatar
Somnath, Suhas committed
865
866
            
            if count == 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
867
                self.__file_handle__.seek(self.__start_point__*4, 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
868
                data_vec = np.fromstring(self.__file_handle__.read(spectrogram_length*4), dtype='f')
869
                pix = BEPSndfPixel(data_vec, self.__wave_type__)
Somnath, Suhas's avatar
Somnath, Suhas committed
870
871
872
873
874
875
876
877
878
879
880
881
882
883
                self.__num_x_steps__ = pix.num_x_steps
                self.__num_y_steps__ = pix.num_y_steps
                self.__num_z_steps__ = pix.num_z_steps
                self.__num_bins__ = pix.num_bins
                
            count += 1
            self.__start_point__ += spectrogram_length
            
            if self.__filesize__ == self.__start_point__*4:
                self.__num_pixels__ = count
                
                # Laser position spectroscopy is NOT accounted for anywhere. 
                # It is impossible to find out from the parms.txt, UD_VS, or the binary .dat file
                num_laser_steps = 1.0*count/(self.__num_z_steps__*self.__num_y_steps__*self.__num_x_steps__)                
Somnath, Suhas's avatar
Somnath, Suhas committed
884
885
886
                if num_laser_steps % 1.0 != 0:
                    print('Some parameter changed inbetween. \
                          BEPS NDF Translator does not handle this usecase at the moment')
Somnath, Suhas's avatar
Somnath, Suhas committed
887
888
889
890
891
892
893
894
895
896
897
898
                else:
                    self.__num_laser_steps__ = int(num_laser_steps)
                
                break
            
            if self.__filesize__ > self.__start_point__*4:
                self.__num_pixels__ = -1
                
        self.__start_point__ = 0
        
        spat_dim = 0
        if self.__num_z_steps__ > 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
899
            # print('Z is varying')
Somnath, Suhas's avatar
Somnath, Suhas committed
900
901
            spat_dim += 1
        if self.__num_y_steps__ > 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
902
            # print('Y is varying')
Somnath, Suhas's avatar
Somnath, Suhas committed
903
904
            spat_dim += 1
        if self.__num_x_steps__ > 1:
Somnath, Suhas's avatar
Somnath, Suhas committed
905
            # print('X is varying')
Somnath, Suhas's avatar
Somnath, Suhas committed
906
907
908
            spat_dim += 1
        if self.__num_laser_steps__ > 1:
            # Laser spot position vector is junk in the .dat file
Somnath, Suhas's avatar
Somnath, Suhas committed
909
            # print('Laser position / unknown parameter varying')
Somnath, Suhas's avatar
Somnath, Suhas committed
910
911
912
            spat_dim += 1
        # print('Total of {} spatial dimensions'.format(spat_dim))
        self.__spat_dim__ = spat_dim
913
914
             
    def read_pixel(self):
Somnath, Suhas's avatar
Somnath, Suhas committed
915
916
917
918
        """
        Returns a BEpixel object containing the parsed information within a pixel.
        Moves pixel index up by one.
        This is where one could conceivably read the file in one pass instead of making 100,000 file I/Os.
919

Somnath, Suhas's avatar
Somnath, Suhas committed
920
        Returns
921
        -------
Somnath, Suhas's avatar
Somnath, Suhas committed
922
923
924
925
926
927
928
929
        pixel : BEPSndfPixel
            Object that describes the data contained within the pixel
        """
        
        if self.__filesize__ == self.__start_point__*4:
            print('BEPS NDF Parser - No more pixels left!')
            return -1
        
930
        self.__file_handle__.seek(self.__start_point__ * 4, 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
931
        spectrogram_length = int(np.fromstring(self.__file_handle__.read(4), dtype='f')[0])  # length of spectrogram
932
        self.__file_handle__.seek(self.__start_point__ * 4, 0)
Somnath, Suhas's avatar
Somnath, Suhas committed
933
934
935
936
937
938
939
940
941
942
        data_vec = np.fromstring(self.__file_handle__.read(spectrogram_length*4), dtype='f')
       
        self.__start_point__ += spectrogram_length
        self.__curr_Pixel__ += 1
                
        if self.__filesize__ == self.__start_point__*4:
            print('BEPS NDF Parser reached End of File')
            self.__EOF__ = True
            self.__file_handle__.close()
                
Somnath, Suhas's avatar
Somnath, Suhas committed
943
        return BEPSndfPixel(data_vec, abs(self.__wave_type__))
Somnath, Suhas's avatar
Somnath, Suhas committed
944
945
946
947
948
        

class BEPSndfPixel(object):
    """
    Stands for BEPS (new data format) Pixel. 
949
950
    This class parses (and keeps) the stream of data contained in a single cell of a BEPS data set of the new data 
    format. Access desired parameter directly without get methods.
Somnath, Suhas's avatar
Somnath, Suhas committed
951
952
    """
    
Somnath, Suhas's avatar
Somnath, Suhas committed
953
    def __init__(self, data_vec, harm=1):
Somnath, Suhas's avatar
Somnath, Suhas committed
954
955
956
957
        """
        Initializes the pixel instance by parsing the provided data. 
        
        Parameters
958
        ----------
Somnath, Suhas's avatar
Somnath, Suhas committed
959
960
961
        data_vec : 1D float numpy array
            Data contained within each pixel
        harm: unsigned int
Chris Smith's avatar
Chris Smith committed
962
963
            Harmonic of the BE waveform. absolute value of the wave type used to normalize the response waveform.

Somnath, Suhas's avatar
Somnath, Suhas committed
964
965
966
967
968
969
970
971
972
973
974
975
976
        """
        
        harm = abs(harm)
        if harm > 3 or harm < 1:
            harm = 1
            warn('Error in BEPSndfPixel: invalid wave type / harmonic provided.')
        
        # Begin parsing data:
        self.spatial_index = int(data_vec[1])-1
        
        self.spectrogram_length = int(data_vec[0]) 
        
        # calculate indices for parsing
Somnath, Suhas's avatar
Somnath, Suhas committed
977
978
        s1 = int(data_vec[2])  # total rows in pixel
        s2 = int(data_vec[3])  # total cols in pixel
Somnath, Suhas's avatar
Somnath, Suhas committed
979
        data_vec1 = data_vec[2:self.spectrogram_length]    
Somnath, Suhas's avatar
Somnath, Suhas committed
980
981
982
983
984
985
        data_mat1 = data_vec1.reshape(s1, s2)
        spect_size1 = int(data_mat1[1, 0])  # total rows in spectrogram set
        self.num_bins = int(spect_size1/2)   # or, len(BE_bin_w)
        self.num_steps = int(data_mat1[1, 1])  # total cols in spectrogram set 
        s3 = int(s1-spect_size1)  # row index of beginning of spectrogram set
        s4 = int(s2-self.num_steps)  # col index of beginning of spectrogram set
Somnath, Suhas's avatar
Somnath, Suhas committed
986
            
Somnath, Suhas's avatar
Somnath, Suhas committed
987
        self.wave_label = data_mat1[2, 0]  # This is useless
988
989
        self.wave_modulation_type = data_mat1[2, 1]  # this is the one with useful information
        # print 'Pixel #',self.spatial_index,' Wave label: ',self.wave_label, ', Wave Type: ', self.wave_modulation_type
Somnath, Suhas's avatar
Somnath, Suhas committed
990
991
        
        # First get the information from the columns:   
992
993
        fft_be_wave_real = data_mat1[s3:s3-0+self.num_bins, 1]  # real part of excitation waveform  
        fft_be_wave_imag = data_mat1[s3+self.num_bins:s3-0+spect_size1, 1]  # imaginary part of excitation waveform  
Somnath, Suhas's avatar
Somnath, Suhas committed
994
        
Somnath, Suhas's avatar
Somnath, Suhas committed
995
996
        """ Though typecasting the combination of the real and imaginary data looks fine in HDFviewer and Spyder, 
        Labview sees such data as an array of clusters having 'r' and 'i' elements """
997
        # self.FFT_BE_wave = np.complex64(fft_be_wave_real + 1j*fft_be_wave_imag) 
Somnath, Suhas's avatar
Somnath, Suhas committed
998
        
Somnath, Suhas's avatar
Somnath, Suhas committed
999
1000
        # complex excitation waveform! due to a problem in the acquisition software, this may not be normalized properly
        self.FFT_BE_wave = np.zeros(self.num_bins, dtype=np.complex64)
For faster browsing, not all history is shown. View entire blame