Merge pull request #235 from pycroscopy/sidpy_reorg

Importing sidpy instead of pyUSID where appropriate

jupyter_notebooks/BE_Processing.ipynb

@@ -26,11 +26,11 @@
 
 ``` python
 # Make sure needed packages are installed and up-to-date
 import sys
 !conda install --yes --prefix {sys.prefix} numpy scipy matplotlib scikit-learn Ipython ipywidgets h5py
-!{sys.executable} -m pip install -U --no-deps pycroscopy  # this will automatically install pyUSID as well
+!{sys.executable} -m pip install -U --no-deps pycroscopy  # this will automatically install sidpy and pyUSID as well
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
@@ -104,11 +104,11 @@
 You can select desired file type by choosing the second option in the pull down menu on the bottom right of the file window
 
 %% Cell type:code id: tags:
 
 ``` python
-input_file_path = usid.io_utils.file_dialog(caption='Select translated .h5 file or raw experiment data',
+input_file_path = sidpy.io.interface_utils.file_dialog(caption='Select translated .h5 file or raw experiment data',
                                           file_filter='Parameters for raw BE data (*.txt *.mat *xls *.xlsx);; \
                                           Translated file (*.h5)')
 
 (data_dir, filename) = os.path.split(input_file_path)
 
@@ -152,17 +152,17 @@
 
 %% Cell type:code id: tags:
 
 ``` python
 print('Datasets and datagroups within the file:\n------------------------------------')
-usid.hdf_utils.print_tree(h5_file)
+sidpy.hdf.hdf_utils.print_tree(h5_file)
  
 print('\nThe main dataset:\n------------------------------------')
 print(h5_main)
 
 print('\nMetadata or attributes in the measurement datagroup\n------------------------------------')
-for key, val in usid.hdf_utils.get_attributes(h5_main.parent.parent).items():
+for key, val in sidpy.hdf.hdf_utils.get_attributes(h5_main.parent.parent).items():
     print('{} : {}'.format(key, val))
 ```
 
 %% Cell type:markdown id: tags:
 
@@ -177,29 +177,29 @@
 pos_labels = h5_main.pos_dim_labels
 print(pos_labels, pos_dims)
 
 h5_meas_grp = h5_main.parent.parent
 
-parm_dict = usid.hdf_utils.get_attributes(h5_meas_grp)
+parm_dict = sidpy.hdf.hdf_utils.get_attributes(h5_meas_grp)
 
-expt_type = usid.hdf_utils.get_attr(h5_file, 'data_type')
+expt_type = sidpy.hdf.hdf_utils.get_attr(h5_file, 'data_type')
 
 is_ckpfm = expt_type == 'cKPFMData'
 if is_ckpfm:
     num_write_steps = parm_dict['VS_num_DC_write_steps']
     num_read_steps = parm_dict['VS_num_read_steps']
     num_fields = 2
     
 if expt_type != 'BELineData':
-    vs_mode = usid.hdf_utils.get_attr(h5_meas_grp, 'VS_mode')
+    vs_mode = sidpy.hdf.hdf_utils.get_attr(h5_meas_grp, 'VS_mode')
     try:
-        field_mode = usid.hdf_utils.get_attr(h5_meas_grp, 'VS_measure_in_field_loops')
+        field_mode = sidpy.hdf.hdf_utils.get_attr(h5_meas_grp, 'VS_measure_in_field_loops')
     except KeyError:
         print('field mode could not be found. Setting to default value')
         field_mode = 'out-of-field'
     try:
-        vs_cycle_frac = usid.hdf_utils.get_attr(h5_meas_grp, 'VS_cycle_fraction')
+        vs_cycle_frac = sidpy.hdf.hdf_utils.get_attr(h5_meas_grp, 'VS_cycle_fraction')
     except KeyError:
         print('VS cycle fraction could not be found. Setting to default value')
         vs_cycle_frac = 'full'
 ```
 

pycroscopy/analysis/be_loop_fitter.py

@@ -20,12 +20,13 @@ from sklearn.cluster import KMeans
 from scipy.optimize import least_squares
 from scipy.cluster.hierarchy import linkage
 from scipy.spatial.distance import pdist
-from pyUSID.io.dtype_utils import stack_real_to_compound, \
+from sidpy.hdf.dtype_utils import stack_real_to_compound, \
     flatten_compound_to_real
+from sidpy.hdf.hdf_utils import get_attr, write_simple_attrs
+from sidpy.proc.comp_utils import get_MPI, recommend_cpu_cores
 from pyUSID.io.hdf_utils import get_unit_values, get_sort_order, \
-    reshape_to_n_dims, get_attr, create_empty_dataset, create_results_group, \
-    write_reduced_anc_dsets, write_simple_attrs, write_main_dataset
-from pyUSID.processing.comp_utils import get_MPI, recommend_cpu_cores
+    reshape_to_n_dims, create_empty_dataset, create_results_group, \
+    write_reduced_anc_dsets, write_main_dataset
 from pyUSID.io.usi_data import USIDataset
 from .utils.be_loop import projectLoop, fit_loop, generate_guess, \
     loop_fit_function, calc_switching_coef_vec, switching32

pycroscopy/analysis/be_sho_fitter.py

@@ -13,9 +13,9 @@ from enum import Enum
 from warnings import warn
 import numpy as np
 from functools import partial
-from pyUSID.io.hdf_utils import write_simple_attrs, \
-    create_results_group, write_reduced_anc_dsets, create_empty_dataset, \
-    write_main_dataset, get_attr
+from sidpy.hdf.hdf_utils import write_simple_attrs, get_attr
+from pyUSID.io.hdf_utils import create_results_group, write_main_dataset, \
+    write_reduced_anc_dsets, create_empty_dataset
 from pyUSID.io.usi_data import USIDataset
 
 from scipy.signal import find_peaks_cwt

pycroscopy/analysis/fitter.py

@@ -14,7 +14,7 @@ from warnings import warn
 import joblib
 from scipy.optimize import least_squares
 
-from pyUSID.processing.comp_utils import recommend_cpu_cores
+from sidpy.proc.comp_utils import recommend_cpu_cores
 from pyUSID.processing.process import Process
 from pyUSID.io.usi_data import USIDataset
 

pycroscopy/analysis/giv_bayesian.py

@@ -12,12 +12,13 @@ Created on Thu Nov 02 11:48:53 2017
 from __future__ import division, print_function, absolute_import, unicode_literals
 
 import numpy as np
-from pyUSID.processing.process import Process
-from pyUSID.processing.comp_utils import parallel_compute
-from pyUSID.io.dtype_utils import stack_real_to_compound
-from pyUSID.io.hdf_utils import write_main_dataset, create_results_group, create_empty_dataset, write_simple_attrs, \
-    print_tree, get_attributes
+from sidpy.proc.comp_utils import parallel_compute
+from sidpy.hdf.dtype_utils import stack_real_to_compound
+from sidpy.hdf.hdf_utils import write_simple_attrs, print_tree, get_attributes
 from pyUSID.io.write_utils import Dimension
+from pyUSID.processing.process import Process
+from pyUSID.io.hdf_utils import write_main_dataset, create_results_group, \
+    create_empty_dataset
 from pyUSID import USIDataset
 from .utils.giv_utils import do_bayesian_inference, bayesian_inference_on_period
 

pycroscopy/analysis/utils/atom_finding.py

@@ -17,8 +17,9 @@ from sklearn.neighbors import KNeighborsClassifier
 import matplotlib.pyplot as plt
 import matplotlib.patches as patches
 
-from pyUSID.processing.comp_utils import recommend_cpu_cores
-from pyUSID.io.dtype_utils import stack_real_to_compound
+from sidpy.proc.comp_utils import recommend_cpu_cores
+from sidpy.hdf.dtype_utils import stack_real_to_compound
+
 from ...io.virtual_data import VirtualDataset, VirtualGroup
 from ...io.hdf_writer import HDFwriter
 

pycroscopy/analysis/utils/giv_utils.py

@@ -14,8 +14,8 @@ import numpy as np
 import matplotlib.pyplot as plt
 from scipy.linalg import sqrtm
 
-from pyUSID.io.hdf_utils import get_auxiliary_datasets
-from pyUSID.viz.plot_utils import set_tick_font_size
+from sidpy.hdf.hdf_utils import get_auxiliary_datasets
+from sidpy.viz.plot_utils import set_tick_font_size
 
 
 def do_bayesian_inference(i_meas, bias, freq, num_x_steps=251, r_extra=110, gam=0.03, e=10.0, sigma=10., sigmaC=1.,

pycroscopy/io/hdf_writer.py

@@ -15,8 +15,11 @@ from time import time, sleep
 from warnings import warn
 import h5py
 
-from pyUSID.io.hdf_utils import assign_group_index, write_simple_attrs
-from pyUSID.io.reg_ref import attempt_reg_ref_build, write_region_references
+from sidpy.hdf.hdf_utils import write_simple_attrs
+from sidpy.hdf.reg_ref import attempt_reg_ref_build, write_region_references
+
+from pyUSID.io.hdf_utils import assign_group_index
+
 from .virtual_data import VirtualGroup, VirtualDataset, VirtualData
 from ..__version__ import version
 

pycroscopy/io/translators/AR_hdf5.py

@@ -10,9 +10,10 @@ import os
 import sys
 import re  # used to get note values
 
-from pyUSID.io.translator import Translator
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs, print_tree, get_attributes
 from pyUSID.io.write_utils import Dimension, make_indices_matrix
-from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset, write_simple_attrs, print_tree, get_attributes
+from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset
 
 if sys.version_info.major == 3:
     unicode = str

pycroscopy/io/translators/be_odf.py

@@ -18,14 +18,16 @@ from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat
 from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
     createSpecVals, requires_conjugate, generate_bipolar_triangular_waveform, \
     infer_bipolar_triangular_fraction_phase, nf32
-from pyUSID.io.reg_ref import write_region_references
-from pyUSID.io.translator import Translator
+
+from sidpy.hdf.hdf_utils import write_simple_attrs
+from sidpy.hdf.reg_ref import write_region_references
+from sidpy.sid import Translator
+from sidpy.proc.comp_utils import get_available_memory
 from pyUSID.io.write_utils import INDICES_DTYPE, VALUES_DTYPE, Dimension, calc_chunks
 from pyUSID.io.hdf_utils import write_ind_val_dsets, write_main_dataset, \
-    create_indexed_group, write_simple_attrs, write_book_keeping_attrs, copy_attributes,\
+    create_indexed_group, write_book_keeping_attrs, copy_attributes,\
     write_reduced_anc_dsets, get_unit_values
 from pyUSID.io.usi_data import USIDataset
-from pyUSID.processing.comp_utils import get_available_memory
 
 if sys.version_info.major == 3:
     unicode = str

pycroscopy/io/translators/be_odf_relaxation.py

@@ -9,16 +9,16 @@ from __future__ import division, print_function, absolute_import, unicode_litera
 
 from os import path, remove  # File Path formatting
 from warnings import warn
-
 import numpy as np  # For array operations
 from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
 import h5py
-
-from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, generatePlotGroups, createSpecVals, maxReadPixels, \
-    nf32
-from pyUSID.io.translator import Translator
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs
 from pyUSID.io.write_utils import INDICES_DTYPE, Dimension
-from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset, write_simple_attrs
+from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset
+
+from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, \
+    generatePlotGroups, createSpecVals, maxReadPixels, nf32
 
 
 class BEodfRelaxationTranslator(Translator):

pycroscopy/io/translators/beps_data_generator.py

@@ -7,11 +7,12 @@ import numpy as np
 from sklearn.utils import gen_batches
 from skimage.measure import block_reduce
 # Pycroscopy imports
-from pyUSID.io.hdf_utils import get_h5_obj_refs, link_as_main, get_attr, copy_dataset, \
-    write_main_dataset, write_simple_attrs, create_indexed_group, create_results_group, write_reduced_anc_dsets
-from pyUSID.io.dtype_utils import stack_real_to_compound
-from pyUSID.io.translator import Translator
-from pyUSID import USIDataset
+from sidpy.hdf.hdf_utils import get_attr, write_simple_attrs
+from sidpy.hdf.dtype_utils import stack_real_to_compound
+from sidpy.sid import Translator
+from pyUSID.io.hdf_utils import link_as_main, copy_dataset, \
+    write_main_dataset, create_indexed_group, create_results_group, \
+    write_reduced_anc_dsets
 from pyUSID.io.write_utils import Dimension, calc_chunks
 from pyUSID.io.image import read_image
 from ...analysis.utils.be_loop import loop_fit_function

pycroscopy/io/translators/beps_ndf.py

@@ -16,12 +16,15 @@ import numpy as np
 import xlrd as xlreader  # To read the UDVS spreadsheet
 from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
 
-from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, parmsToDict, generatePlotGroups, \
-    normalizeBEresponse, createSpecVals, nf32
-from pyUSID.io.translator import Translator
-from pyUSID.io.write_utils import make_indices_matrix, VALUES_DTYPE, INDICES_DTYPE, calc_chunks
-from pyUSID.io.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import get_h5_obj_refs, link_h5_objects_as_attrs
+
+from pyUSID.io.write_utils import make_indices_matrix, VALUES_DTYPE, \
+    INDICES_DTYPE, calc_chunks
 from pyUSID.io.usi_data import USIDataset
+
+from .df_utils.be_utils import trimUDVS, getSpectroscopicParmLabel, \
+    parmsToDict, generatePlotGroups, normalizeBEresponse, createSpecVals, nf32
 from ..hdf_writer import HDFwriter
 from ..virtual_data import VirtualGroup, VirtualDataset
 

pycroscopy/io/translators/bruker_afm.py

@@ -11,9 +11,12 @@ from collections import OrderedDict
 import numpy as np
 import h5py
 
-from pyUSID.io.translator import Translator
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs
+
 from pyUSID.io.write_utils import Dimension
-from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset, write_simple_attrs, write_ind_val_dsets
+from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset, \
+    write_ind_val_dsets
 from .df_utils.base_utils import read_binary_data
 # TODO: Adopt missing aspects / features from https://github.com/paruch-group/distortcorrect/blob/master/afm/filereader/readNanoscope.py
 

pycroscopy/io/translators/df_utils/be_utils.py

@@ -15,12 +15,13 @@ import matplotlib.pyplot as plt
 import numpy as np
 import xlrd as xlreader
 
-from pyUSID.io.hdf_utils import get_auxiliary_datasets, find_dataset, \
-    link_h5_objects_as_attrs, get_attr, create_indexed_group, \
-    write_simple_attrs, write_main_dataset, get_unit_values
+from sidpy.hdf.hdf_utils import write_simple_attrs, link_h5_objects_as_attrs, \
+    get_attr, get_auxiliary_datasets
+from sidpy.proc.comp_utils import get_available_memory, parallel_compute
 
+from pyUSID.io.hdf_utils import find_dataset, create_indexed_group, \
+    write_main_dataset, get_unit_values
 from pyUSID.io.write_utils import create_spec_inds_from_vals, Dimension
-from pyUSID.processing.comp_utils import get_available_memory, parallel_compute
 
 from ....processing.histogram import build_histogram
 from ....analysis.utils.be_sho import SHOestimateGuess

pycroscopy/io/translators/forc_iv.py

@@ -12,9 +12,11 @@ import h5py
 import numpy as np  # For array operations
 from scipy.io import loadmat
 
-from pyUSID.io.translator import Translator
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs
+
 from pyUSID.io.write_utils import Dimension
-from pyUSID.io.hdf_utils import write_main_dataset, create_indexed_group, write_simple_attrs
+from pyUSID.io.hdf_utils import write_main_dataset, create_indexed_group
 
 
 class ForcIVTranslator(Translator):

pycroscopy/io/translators/general_dynamic_mode.py

@@ -13,11 +13,13 @@ import numpy as np  # For array operations
 from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
 import h5py
 
-from .df_utils.gmode_utils import readGmodeParms
-from pyUSID.io.translator import Translator  # Because this class extends the abstract Translator class
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs, link_h5_objects_as_attrs
+
 from pyUSID.io.write_utils import VALUES_DTYPE, Dimension
-from pyUSID.io.hdf_utils import link_h5_objects_as_attrs, create_indexed_group, \
-    write_simple_attrs, write_main_dataset
+from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset
+
+from .df_utils.gmode_utils import readGmodeParms
 
 
 class GDMTranslator(Translator):

pycroscopy/io/translators/general_mode_vs.py

@@ -13,11 +13,13 @@ import numpy as np  # For array operations
 from scipy.io.matlab import loadmat  # To load parameters stored in Matlab .mat file
 import h5py
 
-from .df_utils.gmode_utils import readGmodeParms
-from pyUSID.io.translator import Translator  # Because this class extends the abstract Translator class
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs, link_h5_objects_as_attrs
+
 from pyUSID.io.write_utils import VALUES_DTYPE, Dimension
-from pyUSID.io.hdf_utils import link_h5_objects_as_attrs, create_indexed_group, \
-    write_simple_attrs, write_main_dataset
+from pyUSID.io.hdf_utils import create_indexed_group, write_main_dataset
+
+from .df_utils.gmode_utils import readGmodeParms
 
 
 class GVSTranslator(Translator):
@@ -59,7 +61,7 @@ class GVSTranslator(Translator):
         #parm_dict['freq_sweep_delay'] = np.float(freq_sweep_parms['delay'].item())
         gen_sig = parm_data['genSig']
         #parm_dict['wfm_fix_d_fast'] = np.int32(gen_sig['restrictT'].item())
-        #freq_array = np.float32(parm_data['freqArray'])
+        freq_array = np.float32(parm_data['freqArray'])
 
         # prepare and write spectroscopic values
         samp_rate = parm_dict['IO_down_samp_rate_[Hz]']

pycroscopy/io/translators/gmode_iv.py

@@ -13,9 +13,11 @@ from warnings import warn
 import h5py
 import numpy as np  # For array operations
 
-from pyUSID.io.translator import Translator
+from sidpy.sid import Translator
+from sidpy.hdf.hdf_utils import write_simple_attrs
+
 from pyUSID.io.write_utils import Dimension
-from pyUSID.io.hdf_utils import write_main_dataset, create_indexed_group, write_simple_attrs
+from pyUSID.io.hdf_utils import write_main_dataset, create_indexed_group
 
 
 class GIVTranslator(Translator):