be_loop_fitter.py 49.5 KB
Newer Older
Somnath, Suhas's avatar
Somnath, Suhas committed
1
2
3
4
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 25 11:48:53 2016

5
@author: Suhas Somnath, Chris R. Smith, Rama K. Vasudevan
Somnath, Suhas's avatar
Somnath, Suhas committed
6
7
8

"""

9
from __future__ import division, print_function, absolute_import, unicode_literals
nlaanait's avatar
nlaanait committed
10

Somnath, Suhas's avatar
Somnath, Suhas committed
11
from warnings import warn
nlaanait's avatar
nlaanait committed
12

Somnath, Suhas's avatar
Somnath, Suhas committed
13
import numpy as np
Chris Smith's avatar
Chris Smith committed
14
import scipy
15
16
17
from sklearn.cluster import KMeans
from scipy.cluster.hierarchy import linkage
from scipy.spatial.distance import pdist
18
from .fitter import Fitter
19
from .utils.be_loop import projectLoop, fit_loop, generate_guess, calc_switching_coef_vec, switching32
Somnath, Suhas's avatar
Somnath, Suhas committed
20
from ..processing.tree import ClusterTree
21
from .be_sho_fitter import sho32
Chris Smith's avatar
Chris Smith committed
22
23
from .fit_methods import BE_Fit_Methods
from .optimize import Optimize
24
25
from pyUSID.io.dtype_utils import flatten_compound_to_real, stack_real_to_compound
from pyUSID.io.hdf_utils import copy_region_refs, \
26
27
    get_sort_order, get_dimensionality, reshape_to_n_dims, reshape_from_n_dims, get_attr, \
    create_empty_dataset, create_results_group, write_reduced_spec_dsets, write_simple_attrs, write_main_dataset
28
from pyUSID import USIDataset
Somnath, Suhas's avatar
Somnath, Suhas committed
29

Chris Smith's avatar
Chris Smith committed
30
31
32
'''
Custom dtypes for the datasets created during fitting.
'''
Chris Smith's avatar
Chris Smith committed
33
34
loop_metrics32 = np.dtype({'names': ['Area', 'Centroid x', 'Centroid y', 'Rotation Angle [rad]', 'Offset'],
                           'formats': [np.float32, np.float32, np.float32, np.float32, np.float32]})
35

Chris Smith's avatar
Chris Smith committed
36
37
crit32 = np.dtype({'names': ['AIC_loop', 'BIC_loop', 'AIC_line', 'BIC_line'],
                   'formats': [np.float32, np.float32, np.float32, np.float32]})
Somnath, Suhas's avatar
Somnath, Suhas committed
38

Chris Smith's avatar
Chris Smith committed
39
40
41
field_names = ['a_0', 'a_1', 'a_2', 'a_3', 'a_4', 'b_0', 'b_1', 'b_2', 'b_3', 'R2 Criterion']
loop_fit32 = np.dtype({'names': field_names,
                       'formats': [np.float32 for name in field_names]})
42

Unknown's avatar
Unknown committed
43

44
class BELoopFitter(Fitter):
45
46
    """
    Analysis of Band excitation loops using functional fits
Chris Smith's avatar
Chris Smith committed
47
48
49
50
51
52
53
54
55
56
57
58
59
60
    
    Parameters
    ----------
    h5_main : h5py.Dataset instance
        The dataset over which the analysis will be performed. This dataset should be linked to the spectroscopic
        indices and values, and position indices and values datasets.
    variables : list(string), Default ['Frequency']
        Lists of attributes that h5_main should possess so that it may be analyzed by Model.
    parallel : bool, optional
        Should the parallel implementation of the fitting be used.  Default True.

    Returns
    -------
    None
61
62
63
64
65

    Notes
    -----
    Quantitative mapping of switching behavior in piezoresponse force microscopy, Stephen Jesse, Ho Nyung Lee,
    and Sergei V. Kalinin, Review of Scientific Instruments 77, 073702 (2006); doi: http://dx.doi.org/10.1063/1.2214699
Chris Smith's avatar
Chris Smith committed
66
    
67
    """
Somnath, Suhas's avatar
Somnath, Suhas committed
68

Chris Smith's avatar
Chris Smith committed
69
70
71
72
    def __init__(self, h5_main, variables=None, parallel=True):
        if variables is None:
            variables = ['DC_Offset']

73
        super(BELoopFitter, self).__init__(h5_main, variables, parallel)
Chris Smith's avatar
Chris Smith committed
74

Chris Smith's avatar
Chris Smith committed
75
        self._h5_group = None
76
77
        self.h5_guess_parameters = None
        self.h5_fit_parameters = None
Chris Smith's avatar
Chris Smith committed
78
79
80
        self._sho_spec_inds = None
        self._sho_spec_vals = None  # used only at one location. can remove if deemed unnecessary
        self._met_spec_inds = None
Chris Smith's avatar
Chris Smith committed
81
82
        self._num_forcs = 1
        self._num_forc_repeats = 1
Chris Smith's avatar
Chris Smith committed
83
        self._sho_pos_inds = None
Chris Smith's avatar
Chris Smith committed
84
85
86
        self._current_pos_slice = slice(None)
        self._current_sho_spec_slice = slice(None)
        self._current_met_spec_slice = slice(None)
Unknown's avatar
Unknown committed
87
        self._fit_offset_index = 0
Chris Smith's avatar
Chris Smith committed
88
89
90
        self._sho_all_but_forc_inds = None
        self._sho_all_but_dc_forc_inds = None
        self._met_all_but_forc_inds = None
Chris Smith's avatar
Chris Smith committed
91
        self._current_forc = 0
Unknown's avatar
Unknown committed
92
        self._maxDataChunk = 1
93
        self._fit_dim_name = variables[0]
94

Chris Smith's avatar
Chris Smith committed
95
    def _is_legal(self, h5_main, variables=None):
96
97
98
        """
        Checks whether or not the provided object can be analyzed by this class.

Chris Smith's avatar
Chris Smith committed
99
100
        Parameters
        ----------
101
102
103
104
105
106
        h5_main : h5py.Dataset instance
            The dataset containing the SHO Fit (not necessarily the dataset directly resulting from SHO fit)
            over which the loop projection, guess, and fit will be performed.
        variables : list(string)
            The dimensions needed to be present in the attributes of h5_main to analyze the data with Model.

Chris Smith's avatar
Chris Smith committed
107
        Returns
108
109
110
        -------
        legal : Boolean
            Whether or not this dataset satisfies the necessary conditions for analysis
Chris Smith's avatar
Chris Smith committed
111

112
        """
Chris Smith's avatar
Chris Smith committed
113
114
115
        if variables is None:
            variables = ['DC_Offset']

Unknown's avatar
Unknown committed
116
        file_data_type = get_attr(h5_main.file, 'data_type')
117
118
        meas_grp_name = h5_main.name.split('/')
        h5_meas_grp = h5_main.file[meas_grp_name[1]]
Unknown's avatar
Unknown committed
119
        meas_data_type = get_attr(h5_meas_grp, 'data_type')
120

Unknown's avatar
Unknown committed
121
        if h5_main.dtype != sho32:
122
            warn('Provided dataset is not a SHO results dataset.')
123
124
            return False

Unknown's avatar
Unknown committed
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        # This check is clunky but should account for case differences.  If Python2 support is dropped, simplify with
        # single check using casefold.
        if not (meas_data_type.lower != file_data_type.lower or meas_data_type.upper != file_data_type.upper):
            warn('Mismatch between file and Measurement group data types for the chosen dataset.')
            print('File data type is {}.  The data type for Measurement group {} is {}'.format(file_data_type,
                                                                                               h5_meas_grp.name,
                                                                                               meas_data_type))
            return False

        if file_data_type == 'BEPSData':
            if get_attr(h5_meas_grp, 'VS_mode') not in ['DC modulation mode', 'current mode']:
                warn('Provided dataset is not a DC modulation or current mode BEPS dataset')
                return False
            elif get_attr(h5_meas_grp, 'VS_cycle_fraction') != 'full':
                warn('Provided dataset does not have full cycles')
                return False

        elif file_data_type == 'cKPFMData':
            if get_attr(h5_meas_grp, 'VS_mode') != 'cKPFM':
                warn('Provided dataset has an unsupported VS_mode.')
                return False

147
        return super(BELoopFitter, self)._is_legal(h5_main, variables)
148

149
150
151
    def _set_guess(self, h5_guess):
        """
        Setup to run the fit on an existing guess dataset.  Sets the attributes
Somnath, Suhas's avatar
Somnath, Suhas committed
152
        normally defined during do_guess.
153
154
155
156
157
158
159
160
161
162

        Parameters
        ----------
        h5_guess : h5py.Dataset
            Dataset object containing the guesses

        """
        '''
        Get the Spectroscopic and Position datasets from `self.h5_main`
        '''
Somnath, Suhas's avatar
Somnath, Suhas committed
163
164
165
        self._sho_spec_inds = self.h5_main.h5_spec_inds
        self._sho_spec_vals = self.h5_main.h5_spec_vals
        self._sho_pos_inds = self.h5_main.h5_pos_inds
166
167
168
169

        '''
        Find the Spectroscopic index for the DC_Offset
        '''
170
        fit_ind = np.argwhere(get_attr(self._sho_spec_vals, 'labels') == self._fit_dim_name).squeeze()
Unknown's avatar
Unknown committed
171
172
        self._fit_spec_index = fit_ind
        self._fit_offset_index = 1 + fit_ind
173
174
175
176
177
178
179
180
181
182
183

        '''
        Get the group and projection datasets
        '''
        self._h5_group = h5_guess.parent
        self.h5_projected_loops = self._h5_group['Projected_Loops']
        self.h5_loop_metrics = self._h5_group['Loop_Metrics']
        self._met_spec_inds = self._h5_group['Loop_Metrics_Indices']

        self.h5_guess = h5_guess

Somnath, Suhas's avatar
Somnath, Suhas committed
184
    def do_guess(self, max_mem=None, processors=None, get_loop_parameters=True):
Chris Smith's avatar
Chris Smith committed
185
        """
Chris Smith's avatar
Chris Smith committed
186
187
        Compute the loop projections and the initial guess for the loop parameters.
        
Chris Smith's avatar
Chris Smith committed
188
189
190
        Parameters
        ----------
        processors : uint, optional
191
192
            Number of processors to use for computing. Currently this is a serial operation and this attribute is
            ignored.
193
            Default None, output of psutil.cpu_count - 2 is used
Chris Smith's avatar
Chris Smith committed
194
        max_mem : uint, optional
195
            Memory in MB to use for computation
196
197
198
199
            Default None, available memory from psutil.virtual_memory is used
        get_loop_parameters : bool, optional
            Should the physical loop parameters be calculated after the guess is done
            Default True
Chris Smith's avatar
Chris Smith committed
200
201
202

        Returns
        -------
203
204
        h5_guess : h5py.Dataset object
            h5py dataset containing the guess parameters
Chris Smith's avatar
Chris Smith committed
205
206
207
208
        """

        # Before doing the Guess, we must first project the loops
        self._create_projection_datasets()
Chris Smith's avatar
Chris Smith committed
209
210
        if max_mem is None:
            max_mem = self._maxDataChunk
211
212
213
214
        else:
            max_mem = min(max_mem, self._maxMemoryMB)
            self._maxDataChunk = int(max_mem / self._maxCpus)

Somnath, Suhas's avatar
Somnath, Suhas committed
215
        self._get_sho_chunk_sizes(max_mem)
Somnath, Suhas's avatar
Somnath, Suhas committed
216
        self._create_guess_datasets()
Chris Smith's avatar
Chris Smith committed
217
218

        '''
Chris Smith's avatar
Chris Smith committed
219
        Get the first slice of the sho and loop metrics
Chris Smith's avatar
Chris Smith committed
220
221
222
223
224
225
        '''
        self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                             self.sho_spec_inds_per_forc * (self._current_forc + 1))
        self._current_met_spec_slice = slice(self.metrics_spec_inds_per_forc * self._current_forc,
                                             self.metrics_spec_inds_per_forc * (self._current_forc + 1))

Chris Smith's avatar
Chris Smith committed
226
227
228
        '''
        Get the dc_offset and data_chunk for the first slice
        '''
Somnath, Suhas's avatar
Somnath, Suhas committed
229
230
        self._get_dc_offset()
        self._get_data_chunk()
Chris Smith's avatar
Chris Smith committed
231
232
233
234

        '''
        Loop over positions
        '''
Chris Smith's avatar
Chris Smith committed
235
236
237
238
239
        while self.data is not None:
            # Reshape the SHO
            print('Generating Guesses for FORC {}, and positions {}-{}'.format(self._current_forc,
                                                                               self._start_pos,
                                                                               self._end_pos))
Chris Smith's avatar
Chris Smith committed
240
241
242
            '''
            Reshape the sho data by loops
            '''
Chris Smith's avatar
Chris Smith committed
243
            if len(self._sho_all_but_forc_inds) == 1:
Chris Smith's avatar
Chris Smith committed
244
                # Check for the special case where there is only one loop
Chris Smith's avatar
Chris Smith committed
245
246
247
248
                loops_2d = np.transpose(self.data)
                order_dc_offset_reverse = np.array([1, 0], dtype=np.uint8)
                nd_mat_shape_dc_first = loops_2d.shape
            else:
Somnath, Suhas's avatar
Somnath, Suhas committed
249
                loops_2d, order_dc_offset_reverse, nd_mat_shape_dc_first = self._reshape_sho_matrix(self.data)
Chris Smith's avatar
Chris Smith committed
250

Chris Smith's avatar
Chris Smith committed
251
252
253
            '''
            Do the projection and guess
            '''
Unknown's avatar
Unknown committed
254
255
            projected_loops_2d, loop_metrics_1d = self._project_loop_batch(self.fit_dim_vec, np.transpose(loops_2d))
            guessed_loops = self._guess_loops(self.fit_dim_vec, projected_loops_2d)
Chris Smith's avatar
Chris Smith committed
256
257
258

            # Reshape back
            if len(self._sho_all_but_forc_inds) != 1:
Unknown's avatar
Unknown committed
259
                projected_loops_2d2 = self._reshape_projected_loops_for_h5(projected_loops_2d.T,
Chris Smith's avatar
Chris Smith committed
260
261
                                                                           order_dc_offset_reverse,
                                                                           nd_mat_shape_dc_first)
Chris Smith's avatar
Chris Smith committed
262
263
            else:
                projected_loops_2d2 = projected_loops_2d
Chris Smith's avatar
Chris Smith committed
264

Somnath, Suhas's avatar
Somnath, Suhas committed
265
266
            metrics_2d = self._reshape_results_for_h5(loop_metrics_1d, nd_mat_shape_dc_first)
            guessed_loops_2 = self._reshape_results_for_h5(guessed_loops, nd_mat_shape_dc_first)
Chris Smith's avatar
Chris Smith committed
267
268

            # Store results
Unknown's avatar
Unknown committed
269
            self.h5_projected_loops[self._start_pos:self._end_pos, self._current_sho_spec_slice] = projected_loops_2d2
Chris Smith's avatar
Chris Smith committed
270
            self.h5_loop_metrics[self._start_pos:self._end_pos, self._current_met_spec_slice] = metrics_2d
Chris Smith's avatar
Chris Smith committed
271
            self.h5_guess[self._start_pos:self._end_pos, self._current_met_spec_slice] = guessed_loops_2
Chris Smith's avatar
Chris Smith committed
272

273
            self.h5_main.file.flush()
274

Chris Smith's avatar
Chris Smith committed
275
276
277
            '''
            Change the starting position and get the next chunk of data
            '''
Chris Smith's avatar
Chris Smith committed
278
            self._start_pos = self._end_pos
Somnath, Suhas's avatar
Somnath, Suhas committed
279
            self._get_data_chunk()
Chris Smith's avatar
Chris Smith committed
280

281
282
283
        if get_loop_parameters:
            self.h5_guess_parameters = self.extract_loop_parameters(self.h5_guess)

284
        return USIDataset(self.h5_guess)
Chris Smith's avatar
Chris Smith committed
285

Chris Smith's avatar
Chris Smith committed
286
287
    def do_fit(self, processors=None, max_mem=None, solver_type='least_squares', solver_options=None,
               obj_func=None,
Somnath, Suhas's avatar
Somnath, Suhas committed
288
               get_loop_parameters=True, h5_guess=None):
Chris Smith's avatar
Chris Smith committed
289
        """
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
        Fit the loops

        Parameters
        ----------
        processors : uint, optional
            Number of processors to use for computing. Currently this is a serial operation
            Default None, output of psutil.cpu_count - 2 is used
        max_mem : uint, optional
            Memory in MB to use for computation
            Default None, available memory from psutil.virtual_memory is used
        solver_type : str
            Which solver from scipy.optimize should be used to fit the loops
        solver_options : dict of str
            Parameters to be passed to the solver defined by `solver_type`
        obj_func : dict of str
            Dictionary defining the class and method for the loop residual function as well
            as the parameters to be passed
        get_loop_parameters : bool, optional
            Should the physical loop parameters be calculated after the guess is done
            Default True
310
311
312
        h5_guess : h5py.Dataset
            Existing guess to use as input to fit.
            Default None
313
314
315

        Returns
        -------
316
317
        results: list
            List of the results returned by the solver
Chris Smith's avatar
Chris Smith committed
318

Chris Smith's avatar
Chris Smith committed
319
        """
Chris Smith's avatar
Chris Smith committed
320
321
322
323
        if obj_func is None:
            obj_func = {'class': 'BE_Fit_Methods', 'obj_func': 'BE_LOOP', 'xvals': np.array([])}
        if solver_options is None:
            solver_options = {'jac': '2-point'}
Chris Smith's avatar
Chris Smith committed
324
        '''
325
        Set the number of processors and the amount of RAM to use in the fit
Chris Smith's avatar
Chris Smith committed
326
        '''
Chris Smith's avatar
Chris Smith committed
327
328
329
330
331
332
        if processors is None:
            processors = self._maxCpus
        else:
            processors = min(processors, self._maxCpus)
        if max_mem is None:
            max_mem = self._maxDataChunk
333
334
335
        else:
            max_mem = min(max_mem, self._maxMemoryMB)
            self._maxDataChunk = int(max_mem / self._maxCpus)
Chris Smith's avatar
Chris Smith committed
336

Chris Smith's avatar
Chris Smith committed
337
338
339
        '''
        Ensure that a guess exists
        '''
340
341
342
343
344
345
        if h5_guess is not None:
            self._set_guess(h5_guess)
        elif self.h5_guess is None:
            print("You need to guess before fitting\n")
            return None

Chris Smith's avatar
Chris Smith committed
346
347
348
        '''
        Setup the datasets
        '''
Unknown's avatar
Unknown committed
349
        self._create_fit_datasets()
Somnath, Suhas's avatar
Somnath, Suhas committed
350
        self._get_sho_chunk_sizes(max_mem)
Chris Smith's avatar
Chris Smith committed
351

Chris Smith's avatar
Chris Smith committed
352
353
354
        '''
        Get the dc_vector and the data for the first loop
        '''
Chris Smith's avatar
Chris Smith committed
355
356
357
358
359
360
        self._start_pos = 0
        self._current_forc = 0
        self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                             self.sho_spec_inds_per_forc * (self._current_forc + 1))
        self._current_met_spec_slice = slice(self.metrics_spec_inds_per_forc * self._current_forc,
                                             self.metrics_spec_inds_per_forc * (self._current_forc + 1))
Somnath, Suhas's avatar
Somnath, Suhas committed
361
        self._get_dc_offset()
Somnath, Suhas's avatar
Somnath, Suhas committed
362
        self._get_guess_chunk()
Chris Smith's avatar
Chris Smith committed
363

Chris Smith's avatar
Chris Smith committed
364
365
366
        '''
        Reshape the sho data by loop
        '''
Chris Smith's avatar
Chris Smith committed
367
        if len(self._sho_all_but_forc_inds) == 1:
Chris Smith's avatar
Chris Smith committed
368
            # Check for the special case of a single loop
Chris Smith's avatar
Chris Smith committed
369
370
371
            loops_2d = np.transpose(self.data)
            nd_mat_shape_dc_first = loops_2d.shape
        else:
Somnath, Suhas's avatar
Somnath, Suhas committed
372
            loops_2d, _, nd_mat_shape_dc_first = self._reshape_sho_matrix(self.data)
Chris Smith's avatar
Chris Smith committed
373

Chris Smith's avatar
Chris Smith committed
374
375
376
        '''
        Shift the loops and vdc vector
        '''
Unknown's avatar
Unknown committed
377
        shift_ind, vdc_shifted = self.shift_vdc(self.fit_dim_vec)
Chris Smith's avatar
Chris Smith committed
378
        loops_2d_shifted = np.roll(loops_2d, shift_ind, axis=0).T
379

Chris Smith's avatar
Chris Smith committed
380
381
382
        '''
        Do the fit
        '''
Chris Smith's avatar
Chris Smith committed
383
384
385
        legit_solver = solver_type in scipy.optimize.__dict__.keys()
        legit_obj_func = obj_func['obj_func'] in BE_Fit_Methods().methods
        if legit_solver and legit_obj_func:
Chris Smith's avatar
Chris Smith committed
386
387
            print("Using solver {} and objective function {} to fit your data\n".format(solver_type,
                                                                                        obj_func['obj_func']))
Chris Smith's avatar
Chris Smith committed
388
            while self.data is not None:
Chris Smith's avatar
Chris Smith committed
389
                opt = LoopOptimize(data=loops_2d_shifted, guess=self.guess, parallel=self._parallel)
Chris Smith's avatar
Chris Smith committed
390
                temp = opt.computeFit(processors=processors, solver_type=solver_type, solver_options=solver_options,
391
                                      obj_func={'class': 'BE_Fit_Methods', 'obj_func': 'BE_LOOP', 'xvals': vdc_shifted})
Chris Smith's avatar
Chris Smith committed
392
                # TODO: need a different .reformatResults to process fitting results
Somnath, Suhas's avatar
Somnath, Suhas committed
393
                temp = self._reformat_results(temp, obj_func['obj_func'])
394
                results = self._reshape_results_for_h5(temp, nd_mat_shape_dc_first)
Chris Smith's avatar
Chris Smith committed
395

396
                self.h5_fit[self._start_pos:self._end_pos, self._current_met_spec_slice] = results
397

Chris Smith's avatar
Chris Smith committed
398
                self._start_pos = self._end_pos
Somnath, Suhas's avatar
Somnath, Suhas committed
399
                self._get_guess_chunk()
Chris Smith's avatar
Chris Smith committed
400
401

        elif legit_obj_func:
Somnath, Suhas's avatar
Somnath, Suhas committed
402
            warn('Error: Solver "%s" does not exist!. For additional info see scipy.optimize\n' % solver_type)
Chris Smith's avatar
Chris Smith committed
403
404
            return None
        elif legit_solver:
Somnath, Suhas's avatar
Somnath, Suhas committed
405
            warn('Error: Objective Functions "%s" is not implemented in pycroscopy.analysis.Fit_Methods' %
Chris Smith's avatar
Chris Smith committed
406
407
                 (obj_func['obj_func']))
            return None
Chris Smith's avatar
Chris Smith committed
408

409
410
411
        if get_loop_parameters:
            self.h5_fit_parameters = self.extract_loop_parameters(self.h5_fit)

412
        return USIDataset(self.h5_fit)
413

Somnath, Suhas's avatar
Somnath, Suhas committed
414
415
    @staticmethod
    def extract_loop_parameters(h5_loop_fit, nuc_threshold=0.03):
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
        """
        Method to extract a set of physical loop parameters from a dataset of fit parameters

        Parameters
        ----------
        h5_loop_fit : h5py.Dataset
            Dataset of loop fit parameters
        nuc_threshold : float
            Nucleation threshold to use in calculation physical parameters

        Returns
        -------
        h5_loop_parm : h5py.Dataset
            Dataset of physical parameters
        """
Chris Smith's avatar
Chris Smith committed
431
        dset_name = h5_loop_fit.name.split('/')[-1] + '_Loop_Parameters'
432
        h5_loop_parameters = create_empty_dataset(h5_loop_fit, dtype=switching32,
433
                                                  dset_name=dset_name,
Somnath, Suhas's avatar
Somnath, Suhas committed
434
                                                  new_attrs={'nuc_threshold': nuc_threshold})
435

436
        loop_coef_vec = flatten_compound_to_real(np.reshape(h5_loop_fit, [-1, 1]))
437
438
439
440
        switching_coef_vec = calc_switching_coef_vec(loop_coef_vec, nuc_threshold)

        h5_loop_parameters[:, :] = switching_coef_vec.reshape(h5_loop_fit.shape)

Chris Smith's avatar
Chris Smith committed
441
442
        h5_loop_fit.file.flush()

443
444
        return h5_loop_parameters

Chris Smith's avatar
Chris Smith committed
445
    def _create_projection_datasets(self):
Chris Smith's avatar
Chris Smith committed
446
447
448
449
        """
        Setup the Loop_Fit Group and the loop projection datasets

        """
Chris Smith's avatar
Chris Smith committed
450
        # First grab the spectroscopic indices and values and position indices
Unknown's avatar
Unknown committed
451
452
453
        self._sho_spec_inds = self.h5_main.h5_spec_inds
        self._sho_spec_vals = self.h5_main.h5_spec_vals
        self._sho_pos_inds = self.h5_main.h5_pos_inds
454

455
        fit_dim_ind = self.h5_main.spec_dim_labels.index(self._fit_dim_name)
456

Unknown's avatar
Unknown committed
457
458
        self._fit_spec_index = fit_dim_ind
        self._fit_offset_index = 1 + fit_dim_ind
459

Chris Smith's avatar
Chris Smith committed
460
        # Calculate the number of loops per position
Unknown's avatar
Unknown committed
461
        cycle_start_inds = np.argwhere(self._sho_spec_inds[fit_dim_ind, :] == 0).flatten()
Chris Smith's avatar
Chris Smith committed
462
        tot_cycles = cycle_start_inds.size
463

464
465
466
        # Make the results group
        self._h5_group = create_results_group(self.h5_main, 'Loop_Fit')
        write_simple_attrs(self._h5_group, {'projection_method': 'pycroscopy BE loop model'})
467

468
469
470
471
472
        # Write datasets
        self.h5_projected_loops = create_empty_dataset(self.h5_main, np.float32, 'Projected_Loops',
                                                       h5_group=self._h5_group)

        h5_loop_met_spec_inds, h5_loop_met_spec_vals = write_reduced_spec_dsets(self._h5_group, self._sho_spec_inds,
473
                                                                                self._sho_spec_vals, self._fit_dim_name,
474
475
476
477
478
479
480
481
482
483
                                                                                basename='Loop_Metrics')

        self.h5_loop_metrics = write_main_dataset(self._h5_group, (self.h5_main.shape[0], tot_cycles), 'Loop_Metrics',
                                                  'Metrics', 'compound', None, None, dtype=loop_metrics32,
                                                  h5_pos_inds=self.h5_main.h5_pos_inds,
                                                  h5_pos_vals=self.h5_main.h5_pos_vals,
                                                  h5_spec_inds=h5_loop_met_spec_inds,
                                                  h5_spec_vals=h5_loop_met_spec_vals)

        # Copy region reference:
Somnath, Suhas's avatar
Somnath, Suhas committed
484
485
        copy_region_refs(self.h5_main, self.h5_projected_loops)
        copy_region_refs(self.h5_main, self.h5_loop_metrics)
486

487
        self.h5_main.file.flush()
Chris Smith's avatar
Chris Smith committed
488
        self._met_spec_inds = self.h5_loop_metrics.h5_spec_inds
489
490

        return
Chris Smith's avatar
Chris Smith committed
491

492
    def _get_sho_chunk_sizes(self, max_mem_mb):
493
494
495
496
497
        """
        Calculates the largest number of positions that can be read into memory for a single FORC cycle

        Parameters
        ----------
Somnath, Suhas's avatar
Somnath, Suhas committed
498
        max_mem_mb : unsigned int
499
500
501
502
503
504
505
506
507
508
509
510
511
            Maximum allowable memory in megabytes
        verbose : Boolean (Optional. Default is False)
            Whether or not to print debugging statements

        Returns
        -------
        max_pos : unsigned int
            largest number of positions that can be read into memory for a single FORC cycle
        sho_spec_inds_per_forc : unsigned int
            Number of indices in the SHO spectroscopic table that will be used per read
        metrics_spec_inds_per_forc : unsigned int
            Number of indices in the Loop metrics spectroscopic table that will be used per read
        """
Chris Smith's avatar
Chris Smith committed
512
        # Step 1: Find number of FORC cycles and repeats (if any), DC steps, and number of loops
Chris Smith's avatar
Chris Smith committed
513
        # dc_offset_index = np.argwhere(self._sho_spec_inds.attrs['labels'] == 'DC_Offset').squeeze()
Unknown's avatar
Unknown committed
514
        num_dc_steps = np.unique(self._sho_spec_inds[self._fit_spec_index, :]).size
Unknown's avatar
Unknown committed
515
        all_spec_dims = list(range(self._sho_spec_inds.shape[0]))
Unknown's avatar
Unknown committed
516
        all_spec_dims.remove(self._fit_spec_index)
Chris Smith's avatar
Chris Smith committed
517
518

        # Remove FORC_cycles
Unknown's avatar
Unknown committed
519
        sho_spec_labels = self.h5_main.spec_dim_labels
Chris Smith's avatar
Chris Smith committed
520
521
522
        has_forcs = 'FORC' in sho_spec_labels or 'FORC_Cycle' in sho_spec_labels
        if has_forcs:
            forc_name = 'FORC' if 'FORC' in sho_spec_labels else 'FORC_Cycle'
523
524
525
526
527
            try:
                forc_pos = sho_spec_labels.index(forc_name)
            except Exception:
                raise
            # forc_pos = np.argwhere(sho_spec_labels == forc_name)[0][0]
Chris Smith's avatar
Chris Smith committed
528
            self._num_forcs = np.unique(self._sho_spec_inds[forc_pos]).size
529
            all_spec_dims.remove(forc_pos)
Chris Smith's avatar
Chris Smith committed
530
531
532
533

            # Remove FORC_repeats
            has_forc_repeats = 'FORC_repeat' in sho_spec_labels
            if has_forc_repeats:
534
535
536
537
538
                try:
                    forc_repeat_pos = sho_spec_labels.index('FORC_repeat')
                except Exception:
                    raise
                # forc_repeat_pos = np.argwhere(sho_spec_labels == 'FORC_repeat')[0][0]
Chris Smith's avatar
Chris Smith committed
539
540
541
                self._num_forc_repeats = np.unique(self._sho_spec_inds[forc_repeat_pos]).size
                all_spec_dims.remove(forc_repeat_pos)

542
        # calculate number of loops:
Chris Smith's avatar
Chris Smith committed
543
544
545
546
        if len(all_spec_dims) == 0:
            loop_dims = 1
        else:
            loop_dims = get_dimensionality(self._sho_spec_inds, all_spec_dims)
547
548
549
550
551
552
553
554
        loops_per_forc = np.product(loop_dims)

        # Step 2: Calculate the largest number of FORCS and positions that can be read given memory limits:
        size_per_forc = num_dc_steps * loops_per_forc * len(self.h5_main.dtype) * self.h5_main.dtype[0].itemsize
        """
        How we arrive at the number for the overhead (how many times the size of the data-chunk we will use in memory)
        1 for the original data, 1 for data copied to all children processes, 1 for results, 0.5 for fit, guess, misc
        """
Chris Smith's avatar
Chris Smith committed
555
        mem_overhead = 3.5
Somnath, Suhas's avatar
Somnath, Suhas committed
556
        max_pos = int(max_mem_mb * 1024 ** 2 / (size_per_forc * mem_overhead))
557
        if self._verbose:
Chris Smith's avatar
Chris Smith committed
558
            print('Can read {} of {} pixels given a {} MB memory limit'.format(max_pos,
Chris Smith's avatar
Chris Smith committed
559
560
                                                                               self._sho_pos_inds.shape[0],
                                                                               max_mem_mb))
Chris Smith's avatar
Chris Smith committed
561
        self.max_pos = int(min(self._sho_pos_inds.shape[0], max_pos))
Chris Smith's avatar
Chris Smith committed
562
563
        self.sho_spec_inds_per_forc = int(self._sho_spec_inds.shape[1] / self._num_forcs / self._num_forc_repeats)
        self.metrics_spec_inds_per_forc = int(self._met_spec_inds.shape[1] / self._num_forcs / self._num_forc_repeats)
564
565

        # Step 3: Read allowed chunk
Chris Smith's avatar
Chris Smith committed
566
567
        self._sho_all_but_forc_inds = list(range(self._sho_spec_inds.shape[0]))
        self._met_all_but_forc_inds = list(range(self._met_spec_inds.shape[0]))
Chris Smith's avatar
Chris Smith committed
568
569
        if self._num_forcs > 1:
            self._sho_all_but_forc_inds.remove(forc_pos)
Chris Smith's avatar
Chris Smith committed
570
            met_forc_pos = np.argwhere(get_attr(self._met_spec_inds, 'labels') == forc_name)[0][0]
Chris Smith's avatar
Chris Smith committed
571
            self._met_all_but_forc_inds.remove(met_forc_pos)
572

Chris Smith's avatar
Chris Smith committed
573
574
575
576
577
            if self._num_forc_repeats > 1:
                self._sho_all_but_forc_inds.remove(forc_repeat_pos)
                met_forc_repeat_pos = np.argwhere(get_attr(self._met_spec_inds, 'labels') == 'FORC_repeat')[0][0]
                self._met_all_but_forc_inds.remove(met_forc_repeat_pos)

Chris Smith's avatar
Chris Smith committed
578
        return
579

580
    def _reshape_sho_matrix(self, raw_2d):
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
        """
        Reshapes the raw 2D SHO matrix (as read from the file) to 2D array
        arranged as [instance x points for a single loop]

        Parameters
        ----------
        raw_2d : 2D compound numpy array
            Raw SHO fitted data arranged as [position, data for a single FORC cycle]

        Returns
        -------
        loops_2d : 2D numpy compound array
            SHO fitted data arranged as [instance or position x dc voltage steps]
        order_dc_offset_reverse : tuple
            Order in which the N dimensional data should be transposed to return it to the same format
            as the input data of this function
        nd_mat_shape_dc_first : 1D numpy unsigned int array
            Shape of the N dimensional array that the loops_2d can be turned into.
            Use the order_dc_offset_reverse after this reshape
        """
601
        # step 4: reshape to N dimensions
Somnath, Suhas's avatar
Somnath, Suhas committed
602
        fit_nd, success = reshape_to_n_dims(raw_2d,
Chris Smith's avatar
Chris Smith committed
603
604
                                            h5_pos=None,
                                            h5_spec=self._sho_spec_inds[self._sho_all_but_forc_inds,
Chris Smith's avatar
Chris Smith committed
605
606
                                                                        self._current_sho_spec_slice],
                                            verbose=self._verbose)
607
608
609
        if not success:
            warn('Error - could not reshape provided raw data chunk...')
            return None
Unknown's avatar
Unknown committed
610
611
612

        dim_names_orig = np.hstack(('Positions', np.array(self.h5_main.spec_dim_labels)[self._sho_all_but_forc_inds]))

613
        if self._verbose:
614
615
            print('Shape of N dimensional dataset:', fit_nd.shape)
            print('Dimensions of order:', dim_names_orig)
616

617
        # step 5: Move the voltage dimension to the first dim
Unknown's avatar
Unknown committed
618
619
620
621
        order_dc_outside_nd = [self._fit_offset_index] + list(range(self._fit_offset_index)) + \
                              list(range(self._fit_offset_index + 1, len(fit_nd.shape)))
        order_dc_offset_reverse = list(range(1, self._fit_offset_index + 1)) + [0] + \
                                  list(range(self._fit_offset_index + 1, len(fit_nd.shape)))
622
        fit_nd2 = np.transpose(fit_nd, tuple(order_dc_outside_nd))
623
        dim_names_dc_out = dim_names_orig[order_dc_outside_nd]
624
        if self._verbose:
625
626
            print('originally:', fit_nd.shape, ', after moving DC offset outside:', fit_nd2.shape)
            print('new dim names:', dim_names_dc_out)
627

628
        # step 6: reshape the ND data to 2D arrays
629
        loops_2d = np.reshape(fit_nd2, (fit_nd2.shape[0], -1))
630
        if self._verbose:
631
            print('Loops ready to be projected of shape (Vdc, all other dims besides FORC):', loops_2d.shape)
632

633
        return loops_2d, order_dc_offset_reverse, fit_nd2.shape
634

Chris Smith's avatar
Chris Smith committed
635
    def _reshape_projected_loops_for_h5(self, projected_loops_2d, order_dc_offset_reverse,
636
                                        nd_mat_shape_dc_first):
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
        """
        Reshapes the 2D projected loops to the format such that they can be written to the h5 file

        Parameters
        ----------
        projected_loops_2d : 2D numpy float array
            Projected loops arranged as [instance or position x dc voltage steps]
        order_dc_offset_reverse : tuple of unsigned ints
            Order in which the N dimensional data should be transposed to return it to the format used in h5 files
        nd_mat_shape_dc_first : 1D numpy unsigned int array
            Shape of the N dimensional array that the loops_2d can be turned into.
            We use the order_dc_offset_reverse after this reshape
        verbose : Boolean (Optional. Default is False)
            Whether or not to print debugging statements

        Returns
        -------
        proj_loops_2d : 2D numpy float array
            Projected loops reshaped to the original chronological order in which the data was acquired
        """
657
        if self._verbose:
658
            print('Projected loops of shape:', projected_loops_2d.shape, ', need to bring to:', nd_mat_shape_dc_first)
659
        # Step 9: Reshape back to same shape as fit_Nd2:
Chris Smith's avatar
Chris Smith committed
660
        projected_loops_nd = np.reshape(projected_loops_2d, nd_mat_shape_dc_first)
661
        if self._verbose:
662
            print('Projected loops reshaped to N dimensions :', projected_loops_nd.shape)
663
664
        # Step 10: Move Vdc back inwards. Only for projected loop
        projected_loops_nd_2 = np.transpose(projected_loops_nd, order_dc_offset_reverse)
665
        if self._verbose:
666
            print('Projected loops after moving DC offset inwards:', projected_loops_nd_2.shape)
667
        # step 11: reshape back to 2D
Somnath, Suhas's avatar
Somnath, Suhas committed
668
        proj_loops_2d, success = reshape_from_n_dims(projected_loops_nd_2,
Chris Smith's avatar
Chris Smith committed
669
670
671
                                                     h5_pos=None,
                                                     h5_spec=self._sho_spec_inds[self._sho_all_but_forc_inds,
                                                                                 self._current_sho_spec_slice])
672
673
674
        if not success:
            warn('unable to reshape projected loops')
            return None
675
        if self._verbose:
676
            print('loops shape after collapsing dimensions:', proj_loops_2d.shape)
677
678
679

        return proj_loops_2d

Chris Smith's avatar
Chris Smith committed
680
    def _reshape_results_for_h5(self, raw_results, nd_mat_shape_dc_first):
681
682
        """
        Reshapes the 1D loop metrics to the format such that they can be written to the h5 file
683

684
685
686
687
688
689
690
691
692
693
694
695
696
697
        Parameters
        ----------

        raw_results : 2D numpy float array
            loop metrics arranged as [instance or position x metrics]
        nd_mat_shape_dc_first : 1D numpy unsigned int array
            Shape of the N dimensional array that the raw_results can be turned into.
            We use the order_dc_offset_reverse after this reshape

        Returns
        -------
        metrics_2d : 2D numpy float array
            Loop metrics reshaped to the original chronological order in which the data was acquired
        """
698
        if self._verbose:
699
            print('Loop metrics of shape:', raw_results.shape)
700
        # Step 9: Reshape back to same shape as fit_Nd2:
Chris Smith's avatar
Chris Smith committed
701
        if not self._met_all_but_forc_inds:
Chris Smith's avatar
Chris Smith committed
702
            spec_inds = np.array([[1]])
Chris Smith's avatar
Chris Smith committed
703
704
705
706
707
            loop_metrics_nd = np.reshape(raw_results, [-1, 1])
        else:
            spec_inds = self._met_spec_inds[self._met_all_but_forc_inds, self._current_met_spec_slice]
            loop_metrics_nd = np.reshape(raw_results, nd_mat_shape_dc_first[1:])

708
        if self._verbose:
709
            print('Loop metrics reshaped to N dimensions :', loop_metrics_nd.shape)
Chris Smith's avatar
Chris Smith committed
710

711
        # step 11: reshape back to 2D
Somnath, Suhas's avatar
Somnath, Suhas committed
712
        metrics_2d, success = reshape_from_n_dims(loop_metrics_nd,
Chris Smith's avatar
Chris Smith committed
713
714
                                                  h5_pos=None,
                                                  h5_spec=spec_inds)
715
716
717
        if not success:
            warn('unable to reshape ND results back to 2D')
            return None
718
        if self._verbose:
719
            print('metrics shape after collapsing dimensions:', metrics_2d.shape)
720

721
722
        return metrics_2d

723
    def _get_dc_offset(self):
724
725
        """
        Gets the DC offset for the current FORC step
726

727
728
729
730
        Parameters
        ----------
        verbose : boolean (optional)
            Whether or not to print debugging statements
Somnath, Suhas's avatar
Somnath, Suhas committed
731

732
733
734
735
        Returns
        -------
        dc_vec : 1D float numpy array
            DC offsets for the current FORC step
Somnath, Suhas's avatar
Somnath, Suhas committed
736
        """
737
738
        # apply this knowledge to reshape the spectroscopic values
        # remember to reshape such that the dimensions are arranged in reverse order (slow to fast)
Somnath, Suhas's avatar
Somnath, Suhas committed
739
        spec_vals_nd, success = reshape_to_n_dims(self._sho_spec_vals[self._sho_all_but_forc_inds,
Chris Smith's avatar
Chris Smith committed
740
741
742
                                                                      self._current_sho_spec_slice],
                                                  h5_spec=self._sho_spec_inds[self._sho_all_but_forc_inds,
                                                                              self._current_sho_spec_slice])
743
744
        # This should result in a N+1 dimensional matrix where the first index contains the actual data
        # the other dimensions are present to easily slice the data
Unknown's avatar
Unknown committed
745
        spec_labels_sorted = np.hstack(('Dim', self.h5_main.spec_dim_labels))
746
        if self._verbose:
747
748
749
            print('Spectroscopic dimensions sorted by rate of change:')
            print(spec_labels_sorted)
        # slice the N dimensional dataset such that we only get the DC offset for default values of other dims
750
        fit_dim_pos = np.argwhere(spec_labels_sorted == self._fit_dim_name)[0][0]
Unknown's avatar
Unknown committed
751
752
753
754
755
756
757
758
759
        # fit_dim_slice = list()
        # for dim_ind in range(spec_labels_sorted.size):
        #     if dim_ind == fit_dim_pos:
        #         fit_dim_slice.append(slice(None))
        #     else:
        #         fit_dim_slice.append(slice(0, 1))

        fit_dim_slice = [fit_dim_pos]
        for idim, dim in enumerate(spec_labels_sorted[1:]):
760
            if dim == self._fit_dim_name:
Unknown's avatar
Unknown committed
761
762
                fit_dim_slice.append(slice(None))
                fit_dim_slice[0] = idim
Chris Smith's avatar
Chris Smith committed
763
            elif dim in ['FORC', 'FORC_repeat', 'FORC_Cycle']:
764
                continue
765
            else:
Unknown's avatar
Unknown committed
766
767
                fit_dim_slice.append(slice(0, 1))

768
        if self._verbose:
769
            print('slice to extract Vdc:')
Unknown's avatar
Unknown committed
770
            print(fit_dim_slice)
Chris Smith's avatar
Chris Smith committed
771

Unknown's avatar
Unknown committed
772
        self.fit_dim_vec = np.squeeze(spec_vals_nd[tuple(fit_dim_slice)])
Chris Smith's avatar
Chris Smith committed
773
774

        return
775
776
777

    @staticmethod
    def _project_loop_batch(dc_offset, sho_mat):
778
        """
779
780
781
782
783
        This function projects loops given a matrix of the amplitude and phase.
        These matrices (and the Vdc vector) must have a single cycle's worth of
        points on the second dimension

        Parameters
Chris Smith's avatar
Chris Smith committed
784
        ----------
785
786
787
788
        dc_offset : 1D list or numpy array
            DC voltages. vector of length N
        sho_mat : 2D compound numpy array of type - sho32
            SHO response matrix of size MxN - [pixel, dc voltage]
789

790
        Returns
Chris Smith's avatar
Chris Smith committed
791
        -------
792
793
794
795
796
797
798
799
800
801
802
        results : tuple
            Results from projecting the provided matrices with following components

            projected_loop_mat : MxN numpy array
                Array of Projected loops
            ancillary_mat : M, compound numpy array
                This matrix contains the ancillary information extracted when projecting the loop.
                It contains the following components per loop:
                    'Area' : geometric area of the loop

                    'Centroid x': x positions of centroids for each projected loop
803

804
805
806
807
808
809
810
811
812
                    'Centroid y': y positions of centroids for each projected loop

                    'Rotation Angle': Angle by which loop was rotated [rad]

                    'Offset': Offset removed from loop
        Note
        -----
        This is the function that can be made parallel if need be.
        However, it is fast enough as is
Somnath, Suhas's avatar
Somnath, Suhas committed
813
        """
814
815
        num_pixels = int(sho_mat.shape[0])
        projected_loop_mat = np.zeros(shape=sho_mat.shape, dtype=np.float32)
Chris Smith's avatar
Chris Smith committed
816
        ancillary_mat = np.zeros(shape=num_pixels, dtype=loop_metrics32)
Somnath, Suhas's avatar
Somnath, Suhas committed
817

818
        for pixel in range(num_pixels):
819
820
            """if pixel % 50 == 0:
                print("Projecting Loop {} of {}".format(pixel, num_pixels))"""
Somnath, Suhas's avatar
Somnath, Suhas committed
821

822
823
824
            pix_dict = projectLoop(np.squeeze(dc_offset),
                                   sho_mat[pixel]['Amplitude [V]'],
                                   sho_mat[pixel]['Phase [rad]'])
Somnath, Suhas's avatar
Somnath, Suhas committed
825

826
827
828
829
830
831
            projected_loop_mat[pixel, :] = pix_dict['Projected Loop']
            ancillary_mat[pixel]['Rotation Angle [rad]'] = pix_dict['Rotation Matrix'][0]
            ancillary_mat[pixel]['Offset'] = pix_dict['Rotation Matrix'][1]
            ancillary_mat[pixel]['Area'] = pix_dict['Geometric Area']
            ancillary_mat[pixel]['Centroid x'] = pix_dict['Centroid'][0]
            ancillary_mat[pixel]['Centroid y'] = pix_dict['Centroid'][1]
Somnath, Suhas's avatar
Somnath, Suhas committed
832

833
        return projected_loop_mat, ancillary_mat
Somnath, Suhas's avatar
Somnath, Suhas committed
834

835
    def _project_loops(self):
Chris Smith's avatar
Chris Smith committed
836
        """
Chris Smith's avatar
Chris Smith committed
837
        Do the projection of the SHO fit
Somnath, Suhas's avatar
Somnath, Suhas committed
838
839
840
841
        Parameters
        ----------
        verbose : Boolean
            Whether or not to print debugging statements
Chris Smith's avatar
Chris Smith committed
842
843
844
        """

        self._create_projection_datasets()
845
        self._get_sho_chunk_sizes(10)
Chris Smith's avatar
Chris Smith committed
846
847
848
849
850
851
852
853
854
855
856

        '''
        Loop over the FORCs
        '''
        for forc_chunk_index in range(self._num_forcs):
            pos_chunk_index = 0

            self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                                 self.sho_spec_inds_per_forc * (self._current_forc + 1))
            self._current_met_spec_slice = slice(self.metrics_spec_inds_per_forc * self._current_forc,
                                                 self.metrics_spec_inds_per_forc * (self._current_forc + 1))
857
            dc_vec = self._get_dc_offset()
Chris Smith's avatar
Chris Smith committed
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
            '''
            Loop over positions
            '''
            while self._current_pos_slice.stop < self._end_pos:
                loops_2d, nd_mat_shape_dc_first, order_dc_offset_reverse = self._get_projection_data(pos_chunk_index)

                # step 8: perform loop unfolding
                projected_loops_2d, loop_metrics_1d = self._project_loop_batch(dc_vec, np.transpose(loops_2d))

                # test the reshapes back
                projected_loops_2d = self._reshape_projected_loops_for_h5(projected_loops_2d,
                                                                          order_dc_offset_reverse,
                                                                          nd_mat_shape_dc_first)
                self.h5_projected_loops[self._current_pos_slice, self._current_sho_spec_slice] = projected_loops_2d

                metrics_2d = self._reshape_results_for_h5(loop_metrics_1d, nd_mat_shape_dc_first)

                self.h5_loop_metrics[self._current_pos_slice, self._current_met_spec_slice] = metrics_2d

            # Reset the position slice
            self._current_pos_slice = slice(None)

880
        pass
Somnath, Suhas's avatar
Somnath, Suhas committed
881

882
    def _get_data_chunk(self):
Chris Smith's avatar
Chris Smith committed
883
884
        """
        Get the next chunk of raw data for doing the loop projections.
Somnath, Suhas's avatar
Somnath, Suhas committed
885
886
887
888
889

        Parameters
        ----------
        verbose : Boolean
            Whether or not to print debugging statements
Chris Smith's avatar
Chris Smith committed
890
891
        """
        if self._start_pos < self.max_pos:
Chris Smith's avatar
Chris Smith committed
892
893
            self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                                 self.sho_spec_inds_per_forc * (self._current_forc + 1))
Chris Smith's avatar
Chris Smith committed
894
895
            self._end_pos = int(min(self.h5_main.shape[0], self._start_pos + self.max_pos))
            self.data = self.h5_main[self._start_pos:self._end_pos, self._current_sho_spec_slice]
Unknown's avatar
Unknown committed
896
        elif self._current_forc < self._num_forcs - 1:
Chris Smith's avatar
Chris Smith committed
897
898
899
900
901
902
903
            # Resest for next FORC
            self._current_forc += 1

            self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                                 self.sho_spec_inds_per_forc * (self._current_forc + 1))
            self._current_met_spec_slice = slice(self.metrics_spec_inds_per_forc * self._current_forc,
                                                 self.metrics_spec_inds_per_forc * (self._current_forc + 1))
904
            self._get_dc_offset()
Chris Smith's avatar
Chris Smith committed
905
906
907
908
909
910
911
912
913
914

            self._start_pos = 0
            self._end_pos = int(min(self.h5_main.shape[0], self._start_pos + self.max_pos))
            self.data = self.h5_main[self._start_pos:self._end_pos, self._current_sho_spec_slice]

        else:
            self.data = None

        return

915
    def _get_guess_chunk(self):
Chris Smith's avatar
Chris Smith committed
916
        """
Chris Smith's avatar
Chris Smith committed
917
918
        Read the next chunk of the Guess to use for fitting
        
Somnath, Suhas's avatar
Somnath, Suhas committed
919
920
        Parameters
        ----------
Somnath, Suhas's avatar
Somnath, Suhas committed
921
        verbose : Boolean (optional)
Somnath, Suhas's avatar
Somnath, Suhas committed
922
            Whether or not to print debugging statements
Chris Smith's avatar
Chris Smith committed
923
        """
Chris Smith's avatar
Chris Smith committed
924
925
926
927
928
        if self._start_pos < self.max_pos:
            self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                                 self.sho_spec_inds_per_forc * (self._current_forc + 1))
            self._end_pos = int(min(self.h5_projected_loops.shape[0], self._start_pos + self.max_pos))
            self.data = self.h5_projected_loops[self._start_pos:self._end_pos, self._current_sho_spec_slice]
Unknown's avatar
Unknown committed
929
        elif self._current_forc < self._num_forcs - 1:
Chris Smith's avatar
Chris Smith committed
930
931
932
933
934
935
936
            # Resest for next FORC
            self._current_forc += 1

            self._current_sho_spec_slice = slice(self.sho_spec_inds_per_forc * self._current_forc,
                                                 self.sho_spec_inds_per_forc * (self._current_forc + 1))
            self._current_met_spec_slice = slice(self.metrics_spec_inds_per_forc * self._current_forc,
                                                 self.metrics_spec_inds_per_forc * (self._current_forc + 1))
937
            self._get_dc_offset()
Chris Smith's avatar
Chris Smith committed
938
939
940
941
942
943
944
945
946

            self._start_pos = 0
            self._end_pos = int(min(self.h5_projected_loops.shape[0], self._start_pos + self.max_pos))
            self.data = self.h5_projected_loops[self._start_pos:self._end_pos, self._current_sho_spec_slice]

        else:
            self.data = None

        guess = self.h5_guess[self._start_pos:self._end_pos,
Chris Smith's avatar
Chris Smith committed
947
                self._current_met_spec_slice].reshape([-1, 1])
948
        self.guess = flatten_compound_to_real(guess)[:, :-1]
Chris Smith's avatar
Chris Smith committed
949

Somnath, Suhas's avatar
Somnath, Suhas committed
950
    def _create_guess_datasets(self):
951
952
        """
        Creates the HDF5 Guess dataset and links the it to the ancillary datasets.
953
        """
954
        self.h5_guess = create_empty_dataset(self.h5_loop_metrics, loop_fit32, 'Guess')
955
956
957
958
959
        write_simple_attrs(self._h5_group, {'guess method': 'pycroscopy statistical'})

        # This is necessary comparing against new runs to avoid re-computation + resuming partial computation
        write_simple_attrs(self.h5_guess, self._parms_dict)
        write_simple_attrs(self.h5_guess, {'Loop_fit_method': "pycroscopy statistical", 'last_pixel': 0})
Somnath, Suhas's avatar
Somnath, Suhas committed
960

Chris Smith's avatar
Chris Smith committed
961
        self.h5_main.file.flush()
Chris Smith's avatar
Chris Smith committed
962

963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
    @staticmethod
    def _guess_loops(vdc_vec, projected_loops_2d):
        """
        Provides loop parameter guesses for a given set of loops

        Parameters
        ----------
        vdc_vec : 1D numpy float numpy array
            DC voltage offsets for the loops
        projected_loops_2d : 2D numpy float array
            Projected loops arranged as [instance or position x dc voltage steps]

        Returns
        -------
        guess_parms : 1D compound numpy array
            Loop parameter guesses for the provided projected loops
979
            
980
981
982
983
984
985
986
987
988
989
990