be_utils.py 84 KB
Newer Older
Chris Smith's avatar
Chris Smith committed
1
2
3
4
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 05 07:55:56 2016

5
@author: Suhas Somnath, Chris Smith, Rama K. Vasudevan
Chris Smith's avatar
Chris Smith committed
6
7
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

Chris Smith's avatar
Chris Smith committed
10
11
from os import path
from warnings import warn
12

13
import h5py
Somnath, Suhas's avatar
Somnath, Suhas committed
14
import matplotlib.pyplot as plt
15
import numpy as np
16
import xlrd as xlreader
17

18
19
20
from sidpy.hdf.hdf_utils import write_simple_attrs, link_h5_objects_as_attrs, \
    get_attr, get_auxiliary_datasets
from sidpy.proc.comp_utils import get_available_memory, parallel_compute
21

22
23
from pyUSID.io.hdf_utils import find_dataset, create_indexed_group, \
    write_main_dataset, get_unit_values
24
from pyUSID.io.write_utils import create_spec_inds_from_vals, Dimension
25

Somnath, Suhas's avatar
Somnath, Suhas committed
26
from ....processing.histogram import build_histogram
27
28
29
from ....analysis.utils.be_sho import SHOestimateGuess
from ....viz.be_viz_utils import plot_1d_spectrum, plot_2d_spectrogram, \
    plot_histograms
30
31
from ...hdf_writer import HDFwriter
from ...virtual_data import VirtualDataset, VirtualGroup
32

Chris Smith's avatar
Chris Smith committed
33
34
35
36
nf32 = np.dtype({'names': ['super_band', 'inter_bin_band', 'sub_band'],
                 'formats': [np.float32, np.float32, np.float32]})


37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def generate_bipolar_triangular_waveform(cycle_pts, cycle_frac=1, phase=0,
                                         amplitude=1, cycles=1, offset=0):
    """
    Generates a bi-polar triangular waveform based on basic parameters

    Parameters
    ----------
    cycle_pts : int
        Number of parameters in a single cycle
    cycle_frac : float, optional. Default = 1
        Fraction of the cycle - ranges as (0, 1]
    phase : float, optional. Default = 0
        Phase offset for the waveform. Ranges as [0, 1)
    amplitude : float, optional. Default = 1
        Maximum amplitude for the waveform
    offset : float, optional. Default = 0
        DC offset for the waveform
    cycles : int, optional. Default = 1
        Number of repetitions or cycles

    Returns
    -------
    dc_vec
    """
    # Scale the number of points in the cycle by the (inverse of) the fraction
    actual_cycle_pts = int(cycle_pts // cycle_frac)

    # Build the default bi-polar triangular waveform
    dc_vec = np.hstack(
        (np.linspace(0, 1, actual_cycle_pts // 4, endpoint=False),
         np.linspace(1, 0, actual_cycle_pts // 4, endpoint=False),
         np.linspace(0, -1, actual_cycle_pts // 4, endpoint=False),
         np.linspace(-1, 0, actual_cycle_pts // 4, endpoint=False)))

    # Apply phase offset via a roll:
    dc_vec = np.roll(dc_vec, int(phase * actual_cycle_pts))

    # Next truncate by the fraction
    dc_vec = dc_vec[:cycle_pts]

    # Next, scale by the amplitude:
    dc_vec = amplitude * dc_vec

    # Next offset:
    dc_vec += offset

    # Finally, tile by the number of cycles
    dc_vec = np.tile(dc_vec, int(cycles))

    return dc_vec


89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def infer_bipolar_triangular_fraction_phase(slopes):
    """
    Infers the VS cycle fraction and phase when parameters were
    stored in old mat files

    Parameters
    --------------------
    slopes : list / tuple
        Array of mean slopes of each fraction of a SINGLE cycle

    Returns
    --------------------
    tuple:
        fraction : float
            Fraction of VS cycle
        phase : float
            Phase offset for VS cycle
    """
    if all([_ > 0 for _ in slopes]):
        return 0.25, 0
    elif all([_ < 0 for _ in slopes]):
        return 0.25, 0.75
    elif all([_ > 0 for _ in slopes[:2]]) and all(
            [_ < 0 for _ in slopes[2:]]):
        return 0.5, 0
    elif all([_ < 0 for _ in slopes[:2]]) and all(
            [_ > 0 for _ in slopes[2:]]):
        return 0.5, 0.5
    elif all([_ > 0 for _ in slopes[:1]]) and all(
            [_ < 0 for _ in slopes[1:]]):
        return 0.75, 0
    elif all([_ > 0 for _ in slopes[:3]]) and all(
            [_ < 0 for _ in slopes[3:]]):
        return 0.75, 0.25
    elif all([_ < 0 for _ in slopes[:1]]) and all(
            [_ > 0 for _ in slopes[1:]]):
        return 0.75, 0.5
    elif all([_ < 0 for _ in slopes[:3]]) and all(
            [_ > 0 for _ in slopes[3:]]):
        return 0.75, 0.75
129
    elif slopes[0] > 0 and slopes[1] < 0 and slopes[2] < 0 and slopes[3] > 0:
130
        return 1, 0
131
    elif slopes[0] < 0 and slopes[1] > 0 and slopes[2] > 0 and slopes[3] < 0:
132
133
134
135
136
        return 1, 0.5
    else:
        return 0, 0


137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
def flat_parm_dict_to_nested(parm_dict):
    """
    Creates a nested dictionary of parameters for Band EXcitation from the
    standard flat dictionary

    Parameters
    ----------
    parm_dict : dict
        Dictionary with parameters

    Returns
    -------
    nest_parm_dict : dict
        Nested dictionary of the same parameters
    """
    keys = list(parm_dict.keys())
    keys.sort()
    """
    for key in keys:
        print('{} : {}'.format(key, main_dsets[0].parent.parent.attrs[key]))
    """
    nest_parm_dict = dict()
    for key in ['FORC', 'VS', 'grid', 'BE', 'IO', 'File', 'Misc']:
        nest_parm_dict[key] = dict()
    for key in keys:
        parts = key.split('_')
        parent = 'Misc'
        rem_key = key
        if len(parts) > 1:
            if parts[0] in nest_parm_dict.keys():
                parent = parts[0]
                rem_key = '_'.join(parts[1:])
        nest_parm_dict[parent].update(
            {rem_key: parm_dict[key]})
    return nest_parm_dict


174
175
def remove_non_exist_spec_dim_labs(h5_spec_inds, h5_spec_vals,
                                   h5_meas_grp, verbose=False):
176
177
178
179
180
181
182
183
184
185
186
187
    """
    Removes non-existent spectroscopic dimension name and units from
    attributes of spectroscopic datasets.

    Notes
    -----
    This was written mainly to clean up
    after the buggy Labview HDF5 acquisition software. This will be used in
    the LabviewHDF5Patcher Translator

    Parameters
    ----------
188
189
190
191
192
193
    h5_spec_inds : h5py.Dataset
        Dataset containing the spectroscopic indices
    h5_spec_vals : h5py.Dataset
        Dataset containing the spectroscopic values
    h5_meas_grp : h5py.Group
        Group containing all the parameters for the BE measurement
194
195
196
197
198
199
200
    verbose : bool, optional. Default = False
        Whether or not to print statements aiding in debugging

    Returns
    -------
    None
    """
201
202
203
204
205
206
    for obj, name, exp_type in zip([h5_spec_inds, h5_spec_vals, h5_meas_grp],
                                   ['h5_spec_inds', 'h5_spec_vals', 'h5_meas_grp'],
                                   [h5py.Dataset, h5py.Dataset, h5py.Group]):
        if not isinstance(obj, exp_type):
            raise TypeError('{} should be a {}. Provided object was: {}'
                            ''.format(name, exp_type, type(obj)))
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225

    spec_dim_names = get_attr(h5_spec_inds, 'labels')
    spec_dim_units = get_attr(h5_spec_inds, 'units')
    if len(spec_dim_names) != len(spec_dim_units):
        raise ValueError('Unqeual lengths for the spec dim names and units')
    if len(spec_dim_names) <= h5_spec_inds.shape[0]:
        return

    if verbose:
        print(
            'extra dimensions in the list of names attributes: {} compared to '
            'rows in dataset: {}'
            ''.format(len(spec_dim_names), h5_spec_inds.shape[0]))

    if len(spec_dim_names) - h5_spec_inds.shape[0] > 1:
        raise NotImplementedError(
            'Cannot handle case when more than one dimensions are fake')

    # Gather basic parameters for each dimension
226
    # h5_meas_grp = h5_raw.parent.parent
227
228
229
230
231
232
    field_type = get_attr(h5_meas_grp, 'VS_measure_in_field_loops')
    num_freq_bins = int(get_attr(h5_meas_grp, 'num_bins') /
                        get_attr(h5_meas_grp, 'num_UDVS_steps'))
    num_fields = 1 + int(all([targ in field_type for targ in ['in', 'out']]))
    dc_off_steps = get_attr(h5_meas_grp, 'VS_steps_per_full_cycle')
    num_cycles = get_attr(h5_meas_grp, 'VS_number_of_cycles')
233
234
235
236
237
    try:
        num_forc_cycles = get_attr(h5_meas_grp, 'VS_num_of_FORC_cycles')
    except KeyError:
        # This is named differently in some h5 files
        num_forc_cycles = get_attr(h5_meas_grp, 'VS_num_of_Forc_cycles')
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

    size_dict = {'Frequency': num_freq_bins,
                 'DC_Offset': dc_off_steps,
                 'Field': num_fields,
                 'Cycle': num_cycles,
                 'FORC': num_forc_cycles,
                 }

    if verbose:
        for key, val in size_dict.items():
            print('{} : {}'.format(key, val))

    matched_dims = list()
    matched_units = list()

    row_ind = 0

    for dim_name, dim_units in zip(spec_dim_names, spec_dim_units):
        # Pass one row at a time:
        this_dict = get_unit_values(np.expand_dims(h5_spec_inds[row_ind],
                                                   axis=0),
                                    np.expand_dims(h5_spec_vals[row_ind],
                                                   axis=0),
                                    is_spec=True, all_dim_names=[dim_name])

263
        std_dim_name = dim_name
264
        if dim_name == 'FORC_Cycle':
265
266
            std_dim_name = 'FORC'

267
        if verbose:
268
            print(dim_name, len(this_dict[dim_name]), size_dict[std_dim_name])
269

270
        if len(this_dict[dim_name]) == size_dict[std_dim_name]:
271
272
273
274
275
276
277
278
279
280
            row_ind += 1
            matched_dims.append(dim_name)
            matched_units.append(dim_units)
        else:
            if verbose:
                print(
                    '\tDimension: "' + dim_name + '" did not match with what '
                                                  'was in dataset')

    if verbose:
281
        print('Writing new attributes to spectroscopic datasets')
282
283
284
285
286
287

    new_attrs = {'labels': matched_dims, 'units': matched_units}
    for h5_dset in [h5_spec_inds, h5_spec_vals]:
        write_simple_attrs(h5_dset, new_attrs)


288
def parmsToDict(filepath, parms_to_remove=[]):
Chris Smith's avatar
Chris Smith committed
289
290
291
292
293
294
295
    """
    Translates the parameters in the text file into a dictionary. 
    Also indentifies whether this is a BEPS or BELine dataset.
    
    Parameters
    -----------
    filepath : String / Unicode
296
        Absolute path of the parameters text or spreadsheet file.
Chris Smith's avatar
Chris Smith committed
297
298
299
300
301
302
303
304
305
    parms_to_remove : List of string (Optional)
        keys that this function should attempt to remove from the dictionary
    
    Returns
    ----------
    isBEPS : Boolean
        whether this dataset is BEPS or BE Line.
    parm_dict : Dictionary
        experimental parameters
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
    """
    verbose = False

    lines = list()

    if filepath.lower().endswith('.txt'):
        file_handle = open(filepath, 'r')
        raw_lines = file_handle.readlines()
        file_handle.close()
        for line in raw_lines:
            line = line.rstrip()
            if len(line) > 0:
                lines.append(line.split(" : "))

    elif filepath.lower().endswith('.xls') or filepath.lower().endswith('.xlsx'):
        workbook = xlreader.open_workbook(filepath)
        worksheet = workbook.sheet_by_index(0)
        for row in range(worksheet.nrows):
            temp = list()
            for col in range(worksheet.ncols):
                try:
                    val = str(worksheet.cell(row, col).value).strip()
                    if len(val) > 0:
                        temp.append(val)
                except ValueError:
                    pass
                except:
                    raise
            lines.append(temp)
    else:
        warn('Parameter file not of expected format: text or spreadsheet')
        return None

    if verbose:
        print("Finished reading the file")

    is_beps = False
    parm_dict = dict()

Chris Smith's avatar
Chris Smith committed
345
    prefix = 'File_'
346
    for fields in lines:
Chris Smith's avatar
Chris Smith committed
347
348
349
        # Ignore the parameters describing the GUI choices
        if prefix == 'Multi_':
            continue
350
351

        # Check if the line is a group header or parameter/value pair
Chris Smith's avatar
Chris Smith committed
352
        if len(fields) == 2:
353
354
355
356
357
            # Get the current name/value pair, and clean up the name
            name = fields[0].strip().replace('# ', 'num_').replace('#', 'num_').replace(' ', '_')
            value = fields[1]

            # Rename specific parameters
Chris Smith's avatar
Chris Smith committed
358
359
360
361
            if name == '1_mode':
                name = 'mode'
            if name == 'IO_rate':
                name = 'IO_rate_[Hz]'
Unknown's avatar
Unknown committed
362
                value = int(value.split()[0]) * 1E6
Chris Smith's avatar
Chris Smith committed
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
            if name == 'AO_range':
                name = 'AO_range_[V]'
                value = ' '.join(value.split()[:-1])
            if name == 'AO_amplifier':
                value = value.split()[0]
            if name == 'cycle_time[s]':
                name = 'cycle_time_[s]'
            if name == 'FORC_V_high1[V]':
                name = 'FORC_V_high1_[V]'
            if name == 'FORC_V_high2[V]':
                name = 'FORC_V_high2_[V]'
            if name == 'FORC_V_low1[V]':
                name = 'FORC_V_low1_[V]'
            if name == 'FORC_V_low2[V]':
                name = 'FORC_V_low2_[V]'
            if name == 'amplitude[V]':
                name = 'amplitude_[V]'
            if name == 'offset[V]':
                name = 'offset_[V]'
            if name == 'set_pulst_amplitude[V]':
                name = 'set_pulst_amplitude_[V]'
            if name == 'set_pulst_duration[s]':
                name = 'set_pulst_duration_[s]'
            if name == 'step_edge_smoothing[s]':
                name = 'step_edge_smoothing_[s]'
388
389

            # Append the prefix to the name
Unknown's avatar
Unknown committed
390
            name = prefix + name.lstrip(prefix)
391
392

            # Write parameter to parm_dict
Chris Smith's avatar
Chris Smith committed
393
394
395
396
397
398
399
400
401
402
            try:
                num = float(value)
                parm_dict[name] = num
                if num == int(num):
                    parm_dict[name] = int(num)
            except ValueError:
                parm_dict[name] = value
            except:
                raise
        elif len(fields) == 1:
403
            # Change the parameter prefix to the new one from the group header
Chris Smith's avatar
Chris Smith committed
404
            prefix = fields[0].strip('<').strip('>')
Unknown's avatar
Unknown committed
405
            prefix = prefix.split()[0] + '_'
406
407
408
409
410

            # Check if there are VS Parameters.  Set isBEPS to true if so.
            is_beps = is_beps or prefix == 'VS_'

    if is_beps:
Chris Smith's avatar
Chris Smith committed
411
412
413
414
415
416
417
418
419
420
421
        useless_parms = \
            ['Multi_1_BE_response_spectra',
             'Multi_2_BE_amplitude_spectra',
             'Multi_3_BE_phase_spectra',
             'Multi_4_BE_response_spectrogram',
             'Multi_5_BE_amplitude_spectrogram',
             'Multi_6_BE_phase_spectrogram',
             'Multi_7_VS_response_loops',
             'Multi_8_VS_amplitude_loops',
             'Multi_9_VS_phase_loops',
             'Multi_10_topography',
422
             'Multi_11_mean_channel_2']
Chris Smith's avatar
Chris Smith committed
423
424
425
426
427
428
429
430
431
432
433
434
435
436
    else:
        useless_parms = \
            ['Multi_1_BE_response_spectra',
             'Multi_2_BE_amplitude_spectra',
             'Multi_3_BE_phase_spectra',
             'Multi_4_BE_response_spectrogram',
             'Multi_5_BE_amplitude_spectrogram',
             'Multi_6_BE_phase_spectrogram',
             'Multi_7_amplitude_map',
             'Multi_8_resonance_map',
             'Multi_9_Q_map',
             'Multi_10_phase_map',
             'Multi_11_topography',
             'Multi_12_AI_time_domain',
437
438
439
440
441
442
443
             'Multi_13_AI_Fourier_amplitude']

    if verbose:
        print("Finished parsing the text pairs. isBEPS = {}".format(is_beps))

    useless_parms.extend(parms_to_remove)

Chris Smith's avatar
Chris Smith committed
444
445
446
    # Now remove the list of useless parameters:
    for uparm in useless_parms:
        try:
447
            del parm_dict[uparm]
Chris Smith's avatar
Chris Smith committed
448
        except KeyError:
Unknown's avatar
Unknown committed
449
            # warn('Parameter to be deleted does not exist')
450
            pass
Chris Smith's avatar
Chris Smith committed
451
452
        except:
            raise
453
454
455
456
457
458
    del uparm, useless_parms

    if verbose:
        print("Finished removing useless parameters")

    if is_beps:
Chris Smith's avatar
Chris Smith committed
459
        # fix the DC type in the parms:
460
        if parm_dict['VS_measure_in_field_loops'] == 'out-of-field only':
Chris Smith's avatar
Chris Smith committed
461
            parm_dict['VS_measure_in_field_loops'] = 'out-of-field'
462
        elif parm_dict['VS_measure_in_field_loops'] == 'in-field only':
Chris Smith's avatar
Chris Smith committed
463
            parm_dict['VS_measure_in_field_loops'] = 'in-field'
464
465

    return is_beps, parm_dict
Unknown's avatar
Unknown committed
466
467


Chris Smith's avatar
Chris Smith committed
468
###############################################################################
469
470


471
def requires_conjugate(chosen_spectra, default_q=10, cores=None):
Chris Smith's avatar
Chris Smith committed
472
    """
473
    Determines whether or not the conjugate of the data needs to be taken based on the quality factor
Chris Smith's avatar
Chris Smith committed
474

475
476
477
478
479
480
    Parameters
    ----------
    chosen_spectra : 2D complex numpy array
        N random spectra arranged as [instance, frequency]
    default_q : unsigned int, Optional
        Default value of Q factor that the SHO guess function results in for poor guesses
481
482
483
    cores : uint, Optional
        Number of CPU cores to use for the SHO guess.
        Default = None - ask pyUSID.processing.comp_utils.recommend_cpu_cores for automatic assignment
484
485
486
487
488

    Returns
    -------
    do_conjugate : Boolean
        Whether or not to take the conjugate of the data
Chris Smith's avatar
Chris Smith committed
489
    """
490
    # Do the SHO Guess for each of these
491
492
493
    sho_guess = parallel_compute(chosen_spectra, SHOestimateGuess, cores=cores,
                                 func_args=[np.arange(chosen_spectra.shape[1]),
                                            5])
494

495
    q_results = np.array(sho_guess)[:, 2]
496
497
498
499
500
501
502
503
    good_q = q_results[np.where(q_results != default_q)]

    if np.mean(good_q) < 0:
        return True
    else:
        return False


Chris Smith's avatar
Chris Smith committed
504
505
506
507
508
509
510
###############################################################################

def getSpectroscopicParmLabel(expt_type):
    """
    Returns the label for the spectroscopic parameter in the plot group. 
    
    Parameters
511
512
513
    ----------
    expt_type : str
        Type of the experiment - found in the parms.txt file
Chris Smith's avatar
Chris Smith committed
514
515
    
    Returns
516
517
518
    -------
    str
        label for the spectroscopic parameter axis in the plot
Chris Smith's avatar
Chris Smith committed
519
    """
Unknown's avatar
Unknown committed
520

Somnath, Suhas's avatar
Somnath, Suhas committed
521
    if expt_type in ['DC modulation mode', 'current mode']:
Chris Smith's avatar
Chris Smith committed
522
523
524
525
526
527
528
529
        return 'DC Bias'
    elif expt_type == 'AC modulation mode with time reversal':
        return 'AC amplitude'
    return 'User Defined'


def normalizeBEresponse(spectrogram_mat, FFT_BE_wave, harmonic):
    """
530
    This function normalizes the BE waveform to correct the phase by diving by the excitation
531
    
Chris Smith's avatar
Chris Smith committed
532
533
    Parameters
    ------------
534
535
    spectrogram_mat : 2D complex numpy array
        BE response arranged as [bins, steps]
Chris Smith's avatar
Chris Smith committed
536
    FFT_BE_wave : 1D complex numpy array
537
        FFT of the BE waveform at the appropriate bins. Number of bins must match with spectrogram_mat
Chris Smith's avatar
Chris Smith committed
538
539
    harmonic : unsigned int
        nth harmonic of the excitation waveform
540
        
Chris Smith's avatar
Chris Smith committed
541
    Returns
542
    ----------
Chris Smith's avatar
Chris Smith committed
543
    spectrogram_mat : 2D complex numpy array
544
545
        Normalized BE response spectrogram

Unknown's avatar
Unknown committed
546
    """
547
    BE_wave = np.fft.ifftshift(np.fft.ifft(FFT_BE_wave))
Chris Smith's avatar
Chris Smith committed
548
    scaling_factor = 1
Unknown's avatar
Unknown committed
549

Chris Smith's avatar
Chris Smith committed
550
    if harmonic == 2:
Unknown's avatar
Unknown committed
551
        scaling_factor = np.fft.fftshift(np.fft.fft(BE_wave ** 2)) / (2 * np.exp(1j * 3 * np.pi * 0.5))
Chris Smith's avatar
Chris Smith committed
552
    elif harmonic == 3:
Unknown's avatar
Unknown committed
553
        scaling_factor = np.fft.fftshift(np.fft.fft(BE_wave ** 3)) / (4 * np.exp(1j * np.pi))
554
555
    elif harmonic >= 4:
        print("Warning these high harmonics are not supported in translator.")
Unknown's avatar
Unknown committed
556

557
    # Generate transfer functions
Unknown's avatar
Unknown committed
558
    F_AO_spectrogram = np.transpose(np.tile(FFT_BE_wave / scaling_factor, [spectrogram_mat.shape[1], 1]))
559
    # Divide by transfer function
Unknown's avatar
Unknown committed
560
    spectrogram_mat = spectrogram_mat / F_AO_spectrogram
Unknown's avatar
Unknown committed
561

Chris Smith's avatar
Chris Smith committed
562
    return spectrogram_mat
Somnath, Suhas's avatar
Somnath, Suhas committed
563

Unknown's avatar
Unknown committed
564

565
def generatePlotGroups(h5_main, mean_resp, folder_path, basename, max_resp=[], min_resp=[],
Chris Smith's avatar
Chris Smith committed
566
                       max_mem_mb=1024, spec_label='None',
Unknown's avatar
Unknown committed
567
568
                       show_plots=True, save_plots=True, do_histogram=False,
                       debug=False):
Chris Smith's avatar
Chris Smith committed
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
    """
    Generates the spatially averaged datasets for the given raw dataset. 
    The averaged datasets are necessary for quick visualization of the quality of data. 
    
    Parameters
    ----------
    h5_main : H5 reference
        to the main dataset
    mean_resp : 1D numpy array
        spatially averaged amplitude
    folder_path : String
        Absolute path of the data folder
    basename : String
        base name of the dataset
    max_resp : 1D numpy array
        Maximum amplitude for all pixels
    min_resp : 1D numpy array
        Minimum amplitude for all pixels
    max_mem_mb : Unisigned integer
        Maximum memory that can be used for generating histograms
    spec_label : String
        Parameter that is varying
    show_plots : (optional) Boolean
        Whether or not to show plots
    save_plots : (optional) Boolean
        Whether or not to save generated plots
    do_histogram : Boolean (Optional. Default = False)
        Whether or not to generate hisograms. 
        Caution - Histograms can take a fair amount of time to compute.
Unknown's avatar
Unknown committed
598
599
600
    debug : Boolean, Optional
        If True, then extra debug statements are printed.
        Default False
Chris Smith's avatar
Chris Smith committed
601
    """
602
603
    # Too
    assert isinstance(h5_main, h5py.Dataset)
604
    h5_f = h5_main.file
Chris Smith's avatar
Chris Smith committed
605
606

    grp = h5_main.parent
Unknown's avatar
Unknown committed
607
    h5_freq = grp['Bin_Frequencies']
Chris Smith's avatar
Chris Smith committed
608
    UDVS = grp['UDVS']
609
    spec_inds = h5_main.h5_spec_inds
Chris Smith's avatar
Chris Smith committed
610
    UDVS_inds = grp['UDVS_Indices']
611
    spec_vals = h5_main.h5_spec_vals
Unknown's avatar
Unknown committed
612
613
614

    #     std_cols = ['wave_type','Frequency','DC_Offset','wave_mod','AC_Amplitude','dc_offset','ac_amplitude']

Chris Smith's avatar
Chris Smith committed
615
    col_names = UDVS.attrs['labels']
Unknown's avatar
Unknown committed
616

Chris Smith's avatar
Chris Smith committed
617
618
619
620
621
622
623
    if len(col_names) <= 5:
        """
        No plot groups are defined in the UDVS table.
        All plot group datasets will be written to the 
        Channel group
        """
        return
Unknown's avatar
Unknown committed
624

Chris Smith's avatar
Chris Smith committed
625
    # Removing the standard columns
626
    col_names = get_attr(UDVS, 'labels')[5:]
Chris Smith's avatar
Chris Smith committed
627

Unknown's avatar
Unknown committed
628
629
    #     col_names = [col for col in col_names if col not in std_cols + ignore_plot_groups]

Chris Smith's avatar
Chris Smith committed
630
    freq_inds = spec_inds[spec_inds.attrs['Frequency']].flatten()
Unknown's avatar
Unknown committed
631

Chris Smith's avatar
Chris Smith committed
632
    for col_name in col_names:
633
634
635
636

        # Make sure the column name is a regular Python string
        col_name = str(col_name)

Chris Smith's avatar
Chris Smith committed
637
        ref = UDVS.attrs[col_name]
Unknown's avatar
Unknown committed
638
        #         Make sure we're actually dealing with a reference of some type
Somnath, Suhas's avatar
Somnath, Suhas committed
639
        if not isinstance(ref, h5py.RegionReference):
Chris Smith's avatar
Chris Smith committed
640
            continue
Somnath, Suhas's avatar
Somnath, Suhas committed
641
        # 4. Access that column of the data through region reference
Chris Smith's avatar
Chris Smith committed
642
        steps = np.where(np.isfinite(UDVS[ref]))[0]
643
        step_inds = np.array([np.where(UDVS_inds[()] == step)[0] for step in steps]).flatten()
Chris Smith's avatar
Chris Smith committed
644
645
        """selected_UDVS_steps = UDVS[ref]
        selected_UDVS_steps = selected_UDVS_steps[np.isfinite(selected_UDVS_steps)]"""
Somnath, Suhas's avatar
Somnath, Suhas committed
646
647

        (step_averaged_vec, mean_spec) = reshape_mean_data(spec_inds, step_inds, mean_resp)
Unknown's avatar
Unknown committed
648

Chris Smith's avatar
Chris Smith committed
649
650
651
652
653
654
        """ 
        Need to account for cases with multiple excitation waveforms
        This will affect the frequency indices / values
        We are assuming that there is only one excitation waveform per plot group
        """
        freq_slice = np.unique(freq_inds[step_inds])
655
        freq_vec = h5_freq[()][freq_slice]
Unknown's avatar
Unknown committed
656

Somnath, Suhas's avatar
Somnath, Suhas committed
657
658
        num_bins = len(freq_slice)  # int(len(freq_inds)/len(UDVS[ref]))
        pg_data = np.repeat(UDVS[ref], num_bins)
Unknown's avatar
Unknown committed
659

660
661
        plot_grp = create_indexed_group(grp, 'Spatially_Averaged_Plot_Group')
        write_simple_attrs(plot_grp, {'Name': col_name})
Unknown's avatar
Unknown committed
662

663
664
665
666
667
668
669
670
671
672
673
674
675
676
        h5_mean_spec = plot_grp.create_dataset('Mean_Spectrogram',
                                               data=mean_spec,
                                               dtype=np.complex64)
        h5_step_avg = plot_grp.create_dataset('Step_Averaged_Response',
                                              data=step_averaged_vec,
                                              dtype=np.complex64)
        # cannot assume that this is DC offset, could be AC amplitude....
        h5_spec_parm = plot_grp.create_dataset('Spectroscopic_Parameter',
                                               data=np.squeeze(pg_data[step_inds]),
                                               dtype=np.uint32)
        write_simple_attrs(h5_spec_parm, {'name': spec_label})
        h5_freq_vec = plot_grp.create_dataset('Bin_Frequencies',
                                          data=freq_vec,
                                          dtype=h5_freq.dtype)
Unknown's avatar
Unknown committed
677

Chris Smith's avatar
Chris Smith committed
678
        # Linking the datasets with the frequency and the spectroscopic variable:
Somnath, Suhas's avatar
Somnath, Suhas committed
679
680
        link_h5_objects_as_attrs(h5_mean_spec, [h5_spec_parm, h5_freq_vec])
        link_h5_objects_as_attrs(h5_step_avg, [h5_freq_vec])
Unknown's avatar
Unknown committed
681

Chris Smith's avatar
Chris Smith committed
682
683
684
685
        """
        Create Region Reference for the plot group in the Raw_Data, Spectroscopic_Indices 
        and Spectroscopic_Values Datasets
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
686
687
688
        raw_ref = h5_main.regionref[:, step_inds]
        spec_inds_ref = spec_inds.regionref[:, step_inds]
        spec_vals_ref = spec_vals.regionref[:, step_inds]
Unknown's avatar
Unknown committed
689
690

        ref_name = col_name.replace(' ', '_').replace('-', '_') + '_Plot_Group'
Chris Smith's avatar
Chris Smith committed
691
692
693
        h5_main.attrs[ref_name] = raw_ref
        spec_inds.attrs[ref_name] = spec_inds_ref
        spec_vals.attrs[ref_name] = spec_vals_ref
Unknown's avatar
Unknown committed
694

695
        h5_f.flush()
Unknown's avatar
Unknown committed
696

Chris Smith's avatar
Chris Smith committed
697
698
699
700
701
        if do_histogram:
            """
            Build the histograms for the current plot group
            """
            hist = BEHistogram()
Somnath, Suhas's avatar
Somnath, Suhas committed
702
703
            hist_mat, hist_labels, hist_indices, hist_indices_labels = \
                hist.buildPlotGroupHist(h5_main, step_inds, max_response=max_resp,
Unknown's avatar
Unknown committed
704
                                        min_response=min_resp, max_mem_mb=max_mem_mb, debug=debug)
Chris Smith's avatar
Chris Smith committed
705

706
            hist_grp = create_indexed_group(plot_grp, 'Histogram')
Chris Smith's avatar
Chris Smith committed
707

708
709
710
711
712
713
            hist_spec_dims = list()
            hist_units = ['V', '', 'V', 'V']
            for hist_ind, hist_dim in enumerate(hist_labels):
                hist_spec_dims.append(Dimension(hist_dim,
                                                hist_units[hist_ind],
                                                hist_indices[hist_ind]))
Unknown's avatar
Unknown committed
714

715
            h5_hist = write_main_dataset(hist_grp, hist_mat, 'Histograms',
Unknown's avatar
Unknown committed
716
                                         'Counts', 'a.u.',
717
718
719
720
721
                                         None, hist_spec_dims,
                                         h5_pos_inds=h5_main.h5_pos_inds, h5_pos_vals=h5_main.h5_pos_vals,
                                         dtype=np.int32,
                                         chunking=(1, hist_mat.shape[1]),
                                         compression='gzip')
Unknown's avatar
Unknown committed
722

Chris Smith's avatar
Chris Smith committed
723
724
725
726
        else:
            """
            Write the min and max response vectors so that histograms can be generated later.
            """
727
728
729
730
            h5_max_resp = plot_grp.create_dataset('Max_Response',
                                                  data=max_resp)
            h5_min_resp = plot_grp.create_dataset('Min_Response',
                                                  data=min_resp)
Unknown's avatar
Unknown committed
731

Chris Smith's avatar
Chris Smith committed
732
        if save_plots or show_plots:
Unknown's avatar
Unknown committed
733
            fig_title = '_'.join(grp.name[1:].split('/') + [col_name])
734
735

            fig_1d, axes_1d = plot_1d_spectrum(step_averaged_vec, freq_vec, fig_title)
Chris Smith's avatar
Chris Smith committed
736
            if save_plots:
Somnath, Suhas's avatar
Somnath, Suhas committed
737
738
739
                path_1d = path.join(folder_path, basename + '_Step_Avg_' + fig_title + '.png')
                path_2d = path.join(folder_path, basename + '_Mean_Spec_' + fig_title + '.png')
                path_hist = path.join(folder_path, basename + '_Histograms_' + fig_title + '.png')
740
741

                fig_1d.savefig(path_1d, format='png', dpi=300)
742
743
            if mean_spec.shape[0] > 1:
                fig_2d, axes_2d = plot_2d_spectrogram(mean_spec, freq_vec, title=fig_title)
744
745
746
                if save_plots:
                    fig_2d.savefig(path_2d, format='png', dpi=300)

Chris Smith's avatar
Chris Smith committed
747
            if do_histogram:
748
                plot_histograms(hist_mat, hist_indices, grp.name, figure_path=path_hist)
Unknown's avatar
Unknown committed
749

Chris Smith's avatar
Chris Smith committed
750
751
            if show_plots:
                plt.show()
752
753
            else:
                plt.close('all')
Chris Smith's avatar
Chris Smith committed
754

Unknown's avatar
Unknown committed
755
            # print('Generated spatially average data for group: %s' %(col_name))
Chris Smith's avatar
Chris Smith committed
756
757
    print('Completed generating spatially averaged plot groups')

Unknown's avatar
Unknown committed
758

Chris Smith's avatar
Chris Smith committed
759
###############################################################################
Somnath, Suhas's avatar
Somnath, Suhas committed
760
761
762


def reshape_mean_data(spec_inds, step_inds, mean_resp):
Chris Smith's avatar
Chris Smith committed
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
    """
    Takes in the mean data vector and rearranges that data according to 
    plot group as [step number,bins]
    
    Parameters
    -----------
    spec_inds : 2D numpy array
        UDVS_Indices as a 2D mat
    step_inds : 1D numpy array or list
        UDVS step indices corresponding to this plot group
    mean_resp : 1D numpy complex array
        position averaged BE data
            
    Returns
    ----------
    step_averaged_vec : 1D complex numpy array
        Mean (position averaged) spectrogram averaged over the UDVS steps as well
    mean_spectrogram : 2D complex numpy array
        Position averaged data arranged as [step number,bins]
    """
Somnath, Suhas's avatar
Somnath, Suhas committed
783
    num_bins = len(np.unique(spec_inds[0, step_inds]))
Chris Smith's avatar
Chris Smith committed
784
    # Stephen says that we can assume that the number of bins will NOT change in a plot group
Somnath, Suhas's avatar
Somnath, Suhas committed
785
    mean_spectrogram = mean_resp[step_inds].reshape(-1, num_bins)
Unknown's avatar
Unknown committed
786

Somnath, Suhas's avatar
Somnath, Suhas committed
787
788
    step_averaged_vec = np.mean(mean_spectrogram, axis=0)
    return step_averaged_vec, mean_spectrogram
Unknown's avatar
Unknown committed
789
790


Chris Smith's avatar
Chris Smith committed
791
792
###############################################################################

Somnath, Suhas's avatar
Somnath, Suhas committed
793
794

def visualize_plot_groups(h5_filepath):
Chris Smith's avatar
Chris Smith committed
795
796
797
798
    """
    Visualizes the plot groups present in the provided BE data file
    
    Parameters
Somnath, Suhas's avatar
Somnath, Suhas committed
799
    ----------
Chris Smith's avatar
Chris Smith committed
800
801
802
    h5_filepath : String / Uniciode
        Absolute path of the h5 file
    """
Somnath, Suhas's avatar
Somnath, Suhas committed
803
    with h5py.File(h5_filepath, mode='r') as h5f:
Chris Smith's avatar
Chris Smith committed
804
        expt_type = h5f.attrs.get('data_type')
Somnath, Suhas's avatar
Somnath, Suhas committed
805
        if expt_type not in ['BEPSData', 'BELineData']:
Chris Smith's avatar
Chris Smith committed
806
            warn('Invalid data format')
Unknown's avatar
Unknown committed
807
808
809
810
811
            return
        for grp_name in h5f.keys():
            grp = h5f[grp_name]['Channel_000']
            for plt_grp_name in grp.keys():
                if plt_grp_name.startswith('Spatially_Averaged_Plot_Group_'):
Chris Smith's avatar
Chris Smith committed
812
813
814
                    plt_grp = grp[plt_grp_name]
                    if expt_type == 'BEPSData':
                        spect_data = plt_grp['Mean_Spectrogram'].value
815
816
                        _ = plot_2d_spectrogram(spect_data, plt_grp['Bin_Frequencies'].value,
                                                title=plt_grp.attrs['Name'])
Chris Smith's avatar
Chris Smith committed
817
                    step_avg_data = plt_grp['Step_Averaged_Response']
818
                    _ = plot_1d_spectrum(step_avg_data, plt_grp['Bin_Frequencies'].value, plt_grp.attrs['Name'])
Chris Smith's avatar
Chris Smith committed
819
820
821
                    try:
                        hist_data = plt_grp['Histograms']
                        hist_bins = plt_grp['Histograms_Indicies']
822
                        plot_histograms(hist_data, hist_bins, plt_grp.attrs['Name'])
Chris Smith's avatar
Chris Smith committed
823
824
                    except:
                        pass
Unknown's avatar
Unknown committed
825

Chris Smith's avatar
Chris Smith committed
826
827
    plt.show()
    plt.close('all')
Unknown's avatar
Unknown committed
828
829


Chris Smith's avatar
Chris Smith committed
830
831
###############################################################################

Somnath, Suhas's avatar
Somnath, Suhas committed
832

Chris Smith's avatar
Chris Smith committed
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
def trimUDVS(udvs_mat, udvs_labs, udvs_units, target_col_names):
    """
    Removes unused (typically default) plot groups
    
    Parameters
    ----------
    udvs_mat : 2D numpy array
        UDVS table arranged as [steps, col]
    udvs_labs : list of strings
        Column names of the UDVS table
    udvs_units : list of strings
        Units for the columns of the UDVS table
    target_col_names : list of strings
        Column names that need to be removed
        
    Returns
    -----------
    udvs_mat : 2D numpy array
        Truncated UDVS table
    udvs_labs : list of strings 
        Truncated list of UDVS column names
    udvs_units : list of strings
        Truncated list of UDVS column units
    """
Unknown's avatar
Unknown committed
857

Chris Smith's avatar
Chris Smith committed
858
    if len(target_col_names) == 0:
Somnath, Suhas's avatar
Somnath, Suhas committed
859
        return udvs_mat, udvs_labs, udvs_units
Unknown's avatar
Unknown committed
860

Chris Smith's avatar
Chris Smith committed
861
862
    if len(udvs_labs) != udvs_mat.shape[1]:
        warn('Error: Incompatible UDVS matrix and labels. Not truncating!')
Somnath, Suhas's avatar
Somnath, Suhas committed
863
        return udvs_mat, udvs_labs, udvs_units
Unknown's avatar
Unknown committed
864

Chris Smith's avatar
Chris Smith committed
865
866
867
868
869
870
871
872
873
874
    # First figure out the column indices
    col_inds = []
    found_cols = []
    for target in target_col_names:
        for ind, udvs_col in enumerate(udvs_labs):
            if udvs_col == target:
                col_inds.append(ind)
                found_cols.append(udvs_col)
                break
    col_inds = np.sort(np.unique(col_inds))
Unknown's avatar
Unknown committed
875

Chris Smith's avatar
Chris Smith committed
876
877
    # Now remove from the labels and the matrix
    udvs_mat = np.delete(udvs_mat, col_inds, axis=1)
Somnath, Suhas's avatar
Somnath, Suhas committed
878
879
    # col_inds.sort(reverse=True)
    [udvs_units.pop(ind) for ind in range(len(col_inds), 0, -1)]
Chris Smith's avatar
Chris Smith committed
880
    udvs_labs = [col for col in udvs_labs if col not in found_cols]
Unknown's avatar
Unknown committed
881

Somnath, Suhas's avatar
Somnath, Suhas committed
882
    return udvs_mat, udvs_labs, udvs_units
Chris Smith's avatar
Chris Smith committed
883
884


Unknown's avatar
Unknown committed
885
def createSpecVals(udvs_mat, spec_inds, bin_freqs, bin_wfm_type, parm_dict,
ssomnath's avatar
ssomnath committed
886
                   udvs_labs, udvs_units, verbose=False):
Chris Smith's avatar
Chris Smith committed
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
    """
    This function will determine the proper Spectroscopic Value array for the 
    dataset
    
    Chris Smith -- csmith55@utk.edu
    
    Parameters
    ----------
    udvs_mat : numpy array 
        UDVS table from dataset
    spec_inds : numpy array 
        Spectroscopic Indices table from dataset
    bin_freqs : numpy array 
        Bin frequencies
    bin_wfm_type : numpy array 
        waveform type for each frequency index
    parm_dict : dictionary
        parameters for dataset
    udvs_labs : list of strings
        labels for the columns of the UDVS matrix
    udvs_units : list of strings
        units for the columns of the UDVS matrix
ssomnath's avatar
ssomnath committed
909
910
    verbose : bool, optional. Default = False
        Whether or not to print debugging print statements
Chris Smith's avatar
Chris Smith committed
911
912
913
914
915
916
917
918
919
920
921
922
    
    Returns
    -----------
    ds_spec_val_mat : numpy array 
        Spectroscopic Values table
    ds_spec_val_labs : list of strings
        names of the columns of the Spectroscopic Values table
    ds_spec_val_units : list of strings
        units of the columns of the Spectroscopic Values table
    ds_spec_val_labs_names : list of Strings
        labels with the names of their parameters
    """
Unknown's avatar
Unknown committed
923

924
925
    def __FindSpecValIndices(udvs_mat, spec_inds, usr_defined=False,
                             verbose=False):
Chris Smith's avatar
Chris Smith committed
926
927
928
929
930
931
932
933
934
935
936
937
938
939
        """
        This function finds the Spectroscopic Values associated with the dataset that
        have more than one unique value
        
        Parameters
        ----------
        udvs_mat : numpy array containing the UDVS table
        spec_inds : numpy array contain Spectroscopic indices table
    
        Returns
        -------
        iSpec_var : integer array holding column indices in UDVS that change
        ds_spec_val_mat : array holding all spectral values for columns in iSpec_var
        """
940
941
        udvs_cols = ['step_num', 'dc_offset', 'ac_ampli', 'wave_type',
                     'wave_mod', 'in-field', 'out-of-field']
942
943
944
945
        if verbose:
            print('\t' * 3 + '__FindSpecValIndices:')
            print('\t' * 4 + 'UDVS matrix of shape: {}'.format(udvs_mat.shape))
            print('\t' * 4 + 'spec_inds of shape: {}'.format(spec_inds.shape))
946
947
948
949
            if False:
                print('\t'.join(udvs_cols))
                for ud_row in udvs_mat:
                    print('\t\t'.join(['{:04.2f}'.format(item) for item in ud_row]))
950
            if False:  # Turn this off if necessary
951
952
953
954
                fig, axes = plt.subplots(nrows=2, figsize=(10, 5))
                for ind, axis in enumerate(axes.flat):
                    axis.plot(spec_inds[ind, :])
                    axis.set_title('spec_inds[{}]'.format(ind))
955
                fig.tight_layout()
956

957
958
959
960
        """
        icheck is an array containing all UDVS steps which should be checked.
        """
        icheck = np.unique(spec_inds[1])
961
        if verbose and False:
962
963
964
965
            print('\t' * 4 + 'UDVS steps that will be checked: {}'.format(icheck))
        if len(icheck) < 1:
            raise ValueError('No row in the spectroscopic indices varied.\n'
                             'Cannot build spectroscopic datasets further')
ssomnath's avatar
ssomnath committed
966

967
        # Copy even step values of DC_offset into odd steps
Chris Smith's avatar
Chris Smith committed
968
969
970
        UDVS = np.copy(udvs_mat)

        if not usr_defined:
Somnath, Suhas's avatar
Somnath, Suhas committed
971
972
            DC = UDVS[:, 1]
            for step in range(0, DC.size, 2):
Unknown's avatar
Unknown committed
973
                DC[step + 1] = DC[step]
Somnath, Suhas's avatar
Somnath, Suhas committed
974
            UDVS[:, 1] = DC
Unknown's avatar
Unknown committed
975

Chris Smith's avatar
Chris Smith committed
976
977
978
979
        """
        Keep only the UDVS values for steps which we care about and the 
        first 5 columns
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
980
        UDVS = UDVS[icheck, :5]
981
        udvs_cols = udvs_cols[:5]
Unknown's avatar
Unknown committed
982
        #         UDVS = np.array([UDVS[i] for i in icheck])
983
        if verbose and False:
984
            print('\t' * 4 + 'UDVS matrix after down-selecting rows: {}'
985
                             ''.format(UDVS.shape))
986
            print('\t'.join(udvs_cols))
ssomnath's avatar
ssomnath committed
987
988
            for ud_row in UDVS:
                print('\t\t'.join(['{:04.2f}'.format(item) for item in ud_row]))
Unknown's avatar
Unknown committed
989

Chris Smith's avatar
Chris Smith committed
990
991
992
993
        """
        Transpose UDVS for ease of looping later on and store the number of steps
        as num_cols
        """
Somnath, Suhas's avatar
Somnath, Suhas committed
994
        num_cols = np.size(UDVS, 1)
Chris Smith's avatar
Chris Smith committed
995
996
997
        """
        Initialize the iSpec_var as an empty array.  It will store the index of the 
        UDVS label for any column which has more than one unique value
Unknown's avatar
Unknown committed
998
        """
Chris Smith's avatar
Chris Smith committed
999
        iSpec_var = []
Unknown's avatar
Unknown committed
1000

Chris Smith's avatar
Chris Smith committed
1001
1002
1003
        """
        Loop over all columns in udvs_mat
        """
ssomnath's avatar
ssomnath committed
1004
        for col_ind, col_name in zip(range(1, num_cols), udvs_cols[1:]):
Chris Smith's avatar
Chris Smith committed
1005
1006
1007
            """
            Find all unique values in the current column
            """
1008
1009
1010
            toosmall = np.where(abs(UDVS[:, col_ind]) < 1E-5)[0]
            UDVS[toosmall, col_ind] = 0
            uvals = np.unique(UDVS[:, col_ind])
Chris Smith's avatar
Chris Smith committed
1011
1012
1013
1014
1015
1016
            """
            np.unique considers all NaNs to be unique values
            These two lines find the indices of all NaNs in the unique value array 
            and removes all but the first
            """
            nanvals = np.where(np.isnan(uvals))[0]
1017
1018
1019
            if verbose:
                print('\t\t\tColumn {}: {} has {} unique values and {} NaNs'
                      ''.format(col_ind, col_name, len(uvals), len(nanvals)))
Somnath, Suhas's avatar
Somnath, Suhas committed
1020
            uvals = np.delete(uvals, nanvals[1:])
Chris Smith's avatar
Chris Smith committed
1021
1022
1023
1024
            """
            Check if more that one unique value
            Append column number to iSpec_var if true
            """
1025
1026
1027
            if verbose:
                print('\t\t\tColumn {}: {} had {} actually unique values'
                      ''.format(col_ind, col_name, len(uvals)))
Unknown's avatar
Unknown committed
1028
            if uvals.size > 1:
1029
                iSpec_var = np.append(iSpec_var, int(col_ind))
Unknown's avatar
Unknown committed
1030

1031
1032
1033
1034
1035
        if verbose:
            print('\t' * 4 + 'UDVS matrix of shape: {}'.format(UDVS.shape))
            print('\t' * 4 + 'Taking columns specified by iSpec_var: {}'
                             ''.format(iSpec_var))

1036
1037
1038
        if len(iSpec_var) < 1:
            raise ValueError('Invalid UDVS inputs. No variables were varied')

Somnath, Suhas's avatar
Somnath, Suhas committed
1039
1040
        iSpec_var = np.asarray(iSpec_var, np.int)
        ds_spec_val_mat = UDVS[:, iSpec_var]
Unknown's avatar
Unknown committed
1041

Chris Smith's avatar
Chris Smith committed
1042
        return iSpec_var, ds_spec_val_mat
Unknown's avatar
Unknown committed
1043

ssomnath's avatar
ssomnath committed
1044
1045
    def __BEPSVals(udvs_mat, spec_inds, bin_freqs, bin_wfm_type, parm_dict,
                   udvs_labs, udvs_units, verbose=False):
Chris Smith's avatar
Chris Smith committed
1046
1047
1048
1049
1050
1051
1052
1053
        """
        Returns the Spectroscopic Value array for a BEPS dataset
    
        Parameters
        ----------
        udvs_mat : hdf5 dataset reference to UDVS dataset
        spec_inds : numpy array containing Spectroscopic indices table 
        bin_freqs : 1D numpy array of frequencies
ssomnath's avatar
ssomnath committed
1054
1055
        bin_wfm_type : numpy array containing the waveform type for each
            frequency index
Chris Smith's avatar
Chris Smith committed
1056
1057
1058
1059
1060
1061
1062
1063
        parm_dict : parameter dictinary for dataset
        udvs_labs : list of labels for the columns of the UDVS matrix
        udvs_units : list of units for the columns of the UDVS matrix
        
        Returns
        -------
        ds_spec_val_mat : list holding final Spectroscopic Value table 
        """
Unknown's avatar
Unknown committed
1064

Chris Smith's avatar
Chris Smith committed
1065
1066
1067
1068
        """
        Check the mode to determine if DC, AC, or something else
        """
        mode = parm_dict['VS_mode']
Unknown's avatar
Unknown committed
1069

Chris Smith's avatar
Chris Smith committed
1070
1071
        if mode == 'DC modulation mode' or mode == 'current mode':
            """ 
ssomnath's avatar
ssomnath committed
1072
1073
            First we call the FindSpecVals function to get the columns in UDVS 
            of interest and return a first pass on the spectral value array 
Unknown's avatar
Unknown committed
1074
            """
ssomnath's avatar
ssomnath committed
1075
1076
            if verbose:
                print('\t' * 3 + 'DC modulation mode / current mode')
1077
1078
            iSpecVals, inSpecVals = __FindSpecValIndices(udvs_mat, spec_inds,
                                                         verbose=verbose)
ssomnath's avatar
ssomnath committed
1079
1080
            if verbose:
                print('\t' * 3 + 'Generated spec vals. Calling __BEPSDC')
Unknown's avatar
Unknown committed
1081

ssomnath's avatar
ssomnath committed
1082
            return __BEPSDC(udvs_mat, inSpecVals, bin_freqs, bin_wfm_type,
ssomnath's avatar
ssomnath committed
1083
                            parm_dict, verbose=verbose)
Unknown's avatar
Unknown committed
1084

Chris Smith's avatar
Chris Smith committed
1085
1086
        elif mode == 'AC modulation mode with time reversal':
            """ 
ssomnath's avatar
ssomnath committed
1087
1088
            First we call the FindSpecVals function to get the columns in UDVS
            of interest and return a first pass on the spectral value array 
Unknown's avatar
Unknown committed
1089
            """
ssomnath's avatar
ssomnath committed
1090
1091
            if verbose:
                print('\t' * 3 + 'AC modulation')
1092
1093
            iSpecVals, inSpecVals = __FindSpecValIndices(udvs_mat, spec_inds,
                                                         verbose=verbose)
ssomnath's avatar
ssomnath committed
1094
1095
            if verbose:
                print('\t' * 3 + 'Generated spec vals. Calling __BEPSAC')
Unknown's avatar
Unknown committed
1096

ssomnath's avatar
ssomnath committed
1097
            return __BEPSAC(udvs_mat, inSpecVals, bin_freqs, bin_wfm_type,
ssomnath's avatar
ssomnath committed
1098
                            parm_dict, verbose=verbose)
Chris Smith's avatar
Chris Smith committed
1099
1100
        else:
            """ 
ssomnath's avatar
ssomnath committed
1101
1102
            First we call the FindSpecVals function to get the columns in UDVS
            of interest and return a first pass on the spectral value array 
Unknown's avatar
Unknown committed
1103
            """
ssomnath's avatar
ssomnath committed
1104
1105
1106
            if verbose:
                print('\t' * 3 + 'user defined voltage spectroscopy')
            iSpecVals, inSpecVals = __FindSpecValIndices(udvs_mat, spec_inds,
1107
1108
                                                         usr_defined=True,
                                                         verbose=verbose)
ssomnath's avatar
ssomnath committed
1109
1110
            if verbose:
                print('\t' * 3 + 'Generated spec vals. Calling __BEPSgen')
Unknown's avatar
Unknown committed
1111

Unknown's avatar
Unknown committed
1112
1113
            return __BEPSgen(udvs_mat, inSpecVals, bin_freqs, bin_wfm_type,
                             parm_dict, udvs_labs, iSpecVals, udvs_units)
1114

ssomnath's avatar
ssomnath committed
1115
1116
    def __BEPSDC(udvs_mat, inSpecVals, bin_freqs, bin_wfm_type, parm_dict,
                 verbose=False):
Chris Smith's avatar
Chris Smith committed
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
        """
        Calculates Spectroscopic Values for BEPS data in DC modulation mode
        
        Parameters
        ----------
        udvs_mat : hdf5 dataset reference to UDVS dataset
        inSpecVals : list holding initial guess at spectral values 
        bin_freqs : 1D numpy array of frequencies
        bin_wfm_type : numpy array containing the waveform type for each frequency index
        parm_dict : parameter dictinary for dataset
ssomnath's avatar
ssomnath committed
1127
1128
        verbose: bool, optional. Default = False
            Whether or not to print statements for debugging
Chris Smith's avatar
Chris Smith committed
1129
1130
1131
1132
1133
1134
1135
1136
                        
        Returns
        -------
        ds_spec_val_mat : list holding final Spectroscopic Value table 
        SpecValsLabels : list holding labels of column names of ds_spec_val_mat
        """
        hascycles = False
        hasFORCS = False
Unknown's avatar
Unknown committed
1137

1138
        # print('in shape:',np.shape(inSpecVals))
Chris Smith's avatar
Chris Smith committed
1139
1140
1141
1142
1143
1144
1145
1146
        """
        All DC datasets will need Spectroscopic Value fields for Bin, DC, and Field
        
        Bin     - Bin number
        DC      - dc offset from UDVS
        Field   - 0 if in-field, 1 if out-of-field 
        """
        nrow = 2
Unknown's avatar
Unknown committed
1147
        ds_spec_val_labs = ['Frequency', 'DC_Offset']
Chris Smith's avatar
Chris Smith committed
1148
        ds_spec_val_units = ['Hz', 'V']
Unknown's avatar
Unknown committed
1149

Chris Smith's avatar
Chris Smith committed
1150
1151
1152
1153
        """
        Get the number of and steps in the current dataset
        """
        numsteps = udvs_mat.shape[0]
Unknown's avatar
Unknown committed
1154

Chris Smith's avatar
Chris Smith committed
1155
1156
1157
        """
        Get the wave form for each step from udvs_mat
        """
Unknown's avatar
Unknown committed
1158
1159
        wave_form = udvs_mat[:, 3]

Chris Smith's avatar
Chris Smith committed
1160
1161
1162
1163
1164
1165
1166
1167
        """
        Define list of attribute names needed from the group metadata
        """
        field_type = parm_dict['VS_measure_in_field_loops']
        numcycles = parm_dict['VS_number_of_cycles']
        numFORCs = parm_dict['FORC_num_of_FORC_cycles']
        numcyclesteps = parm_dict['VS_steps_per_full_cycle']
        cycle_fraction = parm_dict['VS_cycle_fraction']
Unknown's avatar
Unknown committed
1168

Chris Smith's avatar
Chris Smith committed
1169
1170
1171
1172
        if field_type == 'in and out-of-field':
            ds_spec_val_labs.append('Field')
            ds_spec_val_units.append('')
            nrow += 1
Unknown's avatar
Unknown committed
1173
1174
1175
1176
1177

        frac = {'full': 1.0, '1/2': 0.5, '1/4': 0.25, '3/4': 0.75}

        numcyclesteps = frac[cycle_fraction] * numcyclesteps

Chris Smith's avatar
Chris Smith committed
1178
1179
1180
1181
1182
1183
1184
1185
1186
        """
        Check the number of cycles and FORCs
        Add to Spectroscopic Values Labels as needed 
        """
        if numcycles > 1:
            hascycles = True
            nrow += 1
            ds_spec_val_labs.append('Cycle')
            ds_spec_val_units.append('')
Unknown's avatar
Unknown committed
1187

Chris Smith's avatar
Chris Smith committed
1188
1189
1190
1191
1192
1193
1194
1195
1196
        if numFORCs > 1:
            hasFORCS = True
            """
        It's possible to have 1 cycle with multiple FORCs so force cycle tracking 
        if FROCs exist
            """
            nrow += 1
            ds_spec_val_labs.append('FORC')
            ds_spec_val_units.append('')
Unknown's avatar
Unknown committed
1197
1198
            numFORCsteps = numcycles * numcyclesteps * 2

Chris Smith's avatar
Chris Smith committed
1199
1200
1201
1202
1203
1204
1205
1206
        """
        Check the field type
        For in-field and out-of-field we know all values of field ahead of time
        For in and out-of-field we must check at each step
        If something else is in field_type, we default to in and out-of-field and print message
        """
        if field_type == 'out-of-field':
            field = 1
Unknown's avatar
Unknown committed
1207
1208
1209
            numsteps = int(numsteps / 2)
            numcyclesteps = int(numcyclesteps / 2)
            swapfield = [1, 1]
Chris Smith's avatar
Chris Smith committed
1210
1211
1212
            field_names = ['out-of-field']
        elif field_type == 'in-field':
            field = 0
Unknown's avatar
Unknown committed
1213
1214
1215
            numsteps = int(numsteps / 2)
            numcyclesteps = int(numcyclesteps / 2)
            swapfield = [0, 0]
Chris Smith's avatar
Chris Smith committed
1216
1217
1218
            field_names = ['in-field']
        elif field_type == 'in and out-of-field':
            field = 0
Unknown's avatar
Unknown committed
1219
1220
            swapfield = [1, 0]
            field_names = ['out-of-field', 'in-field']
Chris Smith's avatar
Chris Smith committed
1221
1222
1223
        else:
            warn('{} is not a known field type'.format(field_type))
            field = 0
Unknown's avatar
Unknown committed