sporc.py 8.36 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
5
6
7
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 29 16:04:34 2016

@author: Suhas Somnath
"""

8
from __future__ import division, print_function, absolute_import, unicode_literals
9

Unknown's avatar
Unknown committed
10
from os import path, remove  # File Path formatting
11

Somnath, Suhas's avatar
Somnath, Suhas committed
12
import h5py
13
14
import numpy as np  # For array operations
from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
15

16
from .translator import Translator  # Because this class extends the abstract Translator class
17
from .utils import make_position_mat, get_position_slicing, generate_dummy_main_parms
18
from ..hdf_utils import getH5DsetRefs, linkRefs
19
from ..io_hdf5 import ioHDF5  # Now the translator is responsible for writing the data.
Unknown's avatar
Unknown committed
20
21
# The building blocks for defining heirarchical storage in the H5 file
from ..microdata import MicroDataGroup, MicroDataset
22

Somnath, Suhas's avatar
Somnath, Suhas committed
23

Somnath, Suhas's avatar
Somnath, Suhas committed
24
class SporcTranslator(Translator):
Somnath, Suhas's avatar
Somnath, Suhas committed
25
26
27
    """
    Translates G-mode SPORC datasets from .mat files to .h5
    """
Unknown's avatar
Unknown committed
28

Unknown's avatar
Unknown committed
29
30
31
    def _read_data(self):
        pass

Unknown's avatar
Unknown committed
32
33
34
    def _parse_file_path(self, input_path):
        pass

Somnath, Suhas's avatar
Somnath, Suhas committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
    def translate(self, parm_path):
        """
        Basic method that translates .mat data files to a single .h5 file
        
        Parameters
        ------------
        parm_path : string / unicode
            Absolute file path of the parameters .mat file. 
            
        Returns
        ----------
        h5_path : string / unicode
            Absolute path of the translated h5 file
        """
Unknown's avatar
Unknown committed
49

Somnath, Suhas's avatar
Somnath, Suhas committed
50
51
        (folder_path, file_name) = path.split(parm_path)
        (file_name, base_name) = path.split(folder_path)
Unknown's avatar
Unknown committed
52
53
        h5_path = path.join(folder_path, base_name + '.h5')

Somnath, Suhas's avatar
Somnath, Suhas committed
54
55
56
57
        # Read parameters
        print('reading parameter files')
        parm_dict, excit_wfm, spec_ind_mat = self.__readparms(parm_path)
        parm_dict['data_type'] = 'SPORC'
Unknown's avatar
Unknown committed
58

Somnath, Suhas's avatar
Somnath, Suhas committed
59
        num_rows = parm_dict['grid_num_rows']
Unknown's avatar
Unknown committed
60
61
62
        num_cols = parm_dict['grid_num_cols']
        num_pix = num_rows * num_cols

63
64
        pos_mat = make_position_mat([num_cols, num_rows])
        pos_slices = get_position_slicing(['X', 'Y'], num_pix)
Unknown's avatar
Unknown committed
65

Somnath, Suhas's avatar
Somnath, Suhas committed
66
67
        # new data format
        spec_ind_mat = np.transpose(np.float32(spec_ind_mat))
Unknown's avatar
Unknown committed
68

Somnath, Suhas's avatar
Somnath, Suhas committed
69
70
71
72
73
        # Now start creating datasets and populating:
        ds_pos_ind = MicroDataset('Position_Indices', np.uint32(pos_mat))
        ds_pos_ind.attrs['labels'] = pos_slices
        ds_pos_val = MicroDataset('Position_Values', np.float32(pos_mat))
        ds_pos_val.attrs['labels'] = pos_slices
Unknown's avatar
Unknown committed
74
75
        ds_pos_val.attrs['units'] = ['um', 'um']

Somnath, Suhas's avatar
Somnath, Suhas committed
76
77
78
        spec_ind_labels = ['x index', 'y index', 'loop index', 'repetition index', 'slope index']
        spec_ind_dict = dict()
        for col_ind, col_name in enumerate(spec_ind_labels):
Unknown's avatar
Unknown committed
79
            spec_ind_dict[col_name] = (slice(col_ind, col_ind + 1), slice(None))
Somnath, Suhas's avatar
Somnath, Suhas committed
80
81
82
83
        ds_spec_inds = MicroDataset('Spectroscopic_Indices', np.uint32(spec_ind_mat))
        ds_spec_inds.attrs['labels'] = spec_ind_dict
        ds_spec_vals = MicroDataset('Spectroscopic_Values', spec_ind_mat)
        ds_spec_vals.attrs['labels'] = spec_ind_dict
Unknown's avatar
Unknown committed
84
        ds_spec_vals.attrs['units'] = ['V', 'V', '', '', '']
Somnath, Suhas's avatar
Somnath, Suhas committed
85
86

        ds_excit_wfm = MicroDataset('Excitation_Waveform', np.float32(excit_wfm))
Unknown's avatar
Unknown committed
87
88
89
90
91
92

        ds_raw_data = MicroDataset('Raw_Data', data=[],
                                   maxshape=(num_pix, len(excit_wfm)),
                                   dtype=np.float16, chunking=(1, len(excit_wfm)),
                                   compression='gzip')

Somnath, Suhas's avatar
Somnath, Suhas committed
93
        # technically should change the date, etc.
Unknown's avatar
Unknown committed
94

Somnath, Suhas's avatar
Somnath, Suhas committed
95
        chan_grp = MicroDataGroup('Channel_000')
Unknown's avatar
Unknown committed
96
        chan_grp.attrs = parm_dict
Somnath, Suhas's avatar
Somnath, Suhas committed
97
98
        chan_grp.addChildren([ds_pos_ind, ds_pos_val, ds_spec_inds, ds_spec_vals,
                              ds_excit_wfm, ds_raw_data])
Unknown's avatar
Unknown committed
99

100
        global_parms = generate_dummy_main_parms()
Unknown's avatar
Unknown committed
101
102
        global_parms['grid_size_x'] = parm_dict['grid_num_cols']
        global_parms['grid_size_y'] = parm_dict['grid_num_rows']
Somnath, Suhas's avatar
Somnath, Suhas committed
103
        # assuming that the experiment was completed:        
Unknown's avatar
Unknown committed
104
105
        global_parms['current_position_x'] = parm_dict['grid_num_cols'] - 1
        global_parms['current_position_y'] = parm_dict['grid_num_rows'] - 1
Unknown's avatar
Unknown committed
106
        global_parms['data_type'] = parm_dict['data_type']
Somnath, Suhas's avatar
Somnath, Suhas committed
107
        global_parms['translator'] = 'SPORC'
Unknown's avatar
Unknown committed
108

Somnath, Suhas's avatar
Somnath, Suhas committed
109
110
111
112
113
        meas_grp = MicroDataGroup('Measurement_000')
        meas_grp.addChildren([chan_grp])
        spm_data = MicroDataGroup('')
        spm_data.attrs = global_parms
        spm_data.addChildren([meas_grp])
Unknown's avatar
Unknown committed
114

Somnath, Suhas's avatar
Somnath, Suhas committed
115
116
        if path.exists(h5_path):
            remove(h5_path)
Unknown's avatar
Unknown committed
117

Somnath, Suhas's avatar
Somnath, Suhas committed
118
119
        # Write everything except for the main data.
        hdf = ioHDF5(h5_path)
Unknown's avatar
Unknown committed
120

Somnath, Suhas's avatar
Somnath, Suhas committed
121
        h5_refs = hdf.writeData(spm_data)
Unknown's avatar
Unknown committed
122

Somnath, Suhas's avatar
Somnath, Suhas committed
123
        h5_main = getH5DsetRefs(['Raw_Data'], h5_refs)[0]
Unknown's avatar
Unknown committed
124
125
126
127

        # Now doing linkrefs:
        aux_ds_names = ['Excitation_Waveform', 'Position_Indices', 'Position_Values',
                        'Spectroscopic_Indices', 'Spectroscopic_Values']
128
        linkRefs(h5_main, getH5DsetRefs(aux_ds_names, h5_refs))
Unknown's avatar
Unknown committed
129

Somnath, Suhas's avatar
Somnath, Suhas committed
130
        print('reading raw data now...')
Unknown's avatar
Unknown committed
131

Somnath, Suhas's avatar
Somnath, Suhas committed
132
133
        # Now read the raw data files:
        pos_ind = 0
Unknown's avatar
Unknown committed
134
135
136
137
        for row_ind in range(1, num_rows + 1):
            for col_ind in range(1, num_cols + 1):
                file_path = path.join(folder_path, 'result_r' + str(row_ind) + '_c' + str(col_ind) + '.mat')
                # print('Working on row {} col {}'.format(row_ind,col_ind))
Somnath, Suhas's avatar
Somnath, Suhas committed
138
139
140
141
142
143
                if path.exists(file_path):
                    # Load data file
                    pix_data = loadmat(file_path, squeeze_me=True)
                    # Take the inverse FFT on 1st dimension
                    pix_vec = np.fft.ifft(np.fft.ifftshift(pix_data['data']))
                    # Verified with Matlab - no conjugate required here.
Unknown's avatar
Unknown committed
144
145
                    h5_main[pos_ind, :] = np.float16(np.real(pix_vec))
                    hdf.flush()  # flush from memory!
Somnath, Suhas's avatar
Somnath, Suhas committed
146
147
                else:
                    print('File for row {} col {} not found'.format(row_ind, col_ind))
Unknown's avatar
Unknown committed
148
                pos_ind += 1
Unknown's avatar
Unknown committed
149
                if (100.0 * pos_ind / num_pix) % 10 == 0:
Unknown's avatar
Unknown committed
150
                    print('Finished reading {} % of data'.format(int(100 * pos_ind / num_pix)))
Somnath, Suhas's avatar
Somnath, Suhas committed
151
152

        hdf.close()
Unknown's avatar
Unknown committed
153

Somnath, Suhas's avatar
Somnath, Suhas committed
154
        return h5_path
Unknown's avatar
Unknown committed
155

Somnath, Suhas's avatar
Somnath, Suhas committed
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
    @staticmethod
    def __readparms(parm_path):
        """
        Copies experimental parameters from the .mat file to a dictionary

        Parameters
        ------------
        parm_path : string / unicode
            Absolute path of the parameters file

        Returns
        --------
        (parm_dict, excit_wfm, spec_ind_mat) : tuple

        parm_dict : dictionary
            Dictionary containing all relevant parameters
        excit_wfm : 1d numpy float array
            Excitation waveform
        spec_ind_mat : 2D numpy flaot array
            Spectroscopic indicies matrix
        """

        parm_data = loadmat(parm_path, squeeze_me=True, struct_as_record=True)
        parm_dict = dict()

        IO_parms = parm_data['IOparms']
        parm_dict['IO_samp_rate_[Hz]'] = np.int32(IO_parms['sampRate'].item())
        parm_dict['IO_down_samp_rate_[Hz]'] = np.int32(IO_parms['downSampRate'].item())
        parm_dict['IO_AO0_amp'] = np.int32(IO_parms['AO0_amp'].item())
        parm_dict['IO_AI_chans'] = np.int32(parm_data['aiChans'])

        parm_dict['grid_num_rows'] = parm_data['numrows']
        parm_dict['grid_num_cols'] = parm_data['numcols']

        sporc_parms = parm_data['sporcParms']
        parm_dict['SPORC_V_max_[V]'] = np.float32(sporc_parms['V_max'].item())
        parm_dict['SPORC_N_steps'] = np.int32(sporc_parms['N_steps'].item())
        parm_dict['SPORC_N_reps'] = np.int32(sporc_parms['N_reps'].item())
        parm_dict['SPORC_t_max_[sec]'] = np.float32(sporc_parms['t_max'])
        parm_dict['SPORC_f_cutoff_[Hz]'] = np.int32(sporc_parms['f_cutoff'])
        parm_dict['SPORC_f_rolloff_[Hz]'] = np.int32(sporc_parms['f_rolloff'])

        if 'FORC_vec' in parm_data.keys() and 'ind_vecs' in parm_data.keys():
            excit_wfm = np.squeeze(np.float32(parm_data['FORC_vec']))
            spec_ind_mat = np.transpose(np.float32(parm_data['ind_vecs']))
        else:
            # Look for a second parms file that contains these vectors:
            fold, basename = path.split(parm_path)
Unknown's avatar
Unknown committed
204
205
            second_path = path.join(fold, 'SPORC_wave.mat')
            h5_sporc_parms = h5py.File(second_path, 'r')  # Use this for v7.3 and beyond.
Somnath, Suhas's avatar
Somnath, Suhas committed
206
207
208
209
            excit_wfm = np.squeeze(h5_sporc_parms['FORC_vec'].value)
            spec_ind_mat = np.float32(h5_sporc_parms['ind_vecs'].value)
            h5_sporc_parms.close()

Unknown's avatar
Unknown committed
210
        return parm_dict, excit_wfm, spec_ind_mat