Commit 6c04eba2 authored by syz's avatar syz
Browse files

Updated to use the new FFT filtering process + minor improvements

parent fc5e801f
%% Cell type:markdown id: tags:
# G-Mode filtering and inspection using pycroscopy
### Suhas Somnath and Stephen Jesse
The Center for Nanophase Materials Science and The Institute for Functional Imaging for Materials <br>
Oak Ridge National Laboratory<br>
5/05/2017
%% Cell type:markdown id: tags:
## Configure the notebook
%% Cell type:code id: tags:
``` python
# Ensure python 3 compatibility
from __future__ import division, print_function, absolute_import, unicode_literals
# Import necessary libraries:
# General utilities:
from os import path
# Computation:
import numpy as np
import h5py
# Visualization:
import matplotlib.pyplot as plt
# Finally, pycroscopy itself
import pycroscopy as px
# set up notebook to show plots within the notebook
% matplotlib inline
```
%% Cell type:code id: tags:
``` python
ui_file_window = False
try:
from PyQt5 import QtWidgets
def uiGetFile(filter='H5 file (*.h5)', caption='Select File'):
"""
Presents a File dialog used for selecting the .mat file
and returns the absolute filepath of the selecte file\n
Parameters
----------
extension : String or list of strings
file extensions to look for
caption : (Optional) String
Title for the file browser window
Returns
-------
file_path : String
Absolute path of the chosen file
"""
app = QtWidgets.QApplication([])
path = QtWidgets.QFileDialog.getOpenFileName(caption=caption, filter=filter)[0]
app.exit()
del app
return str(path)
ui_file_window = True
except ImportError:
print('***********************************************************')
print('* *')
print('* You will need to specify the file path manually below *')
print('* *')
print('***********************************************************')
save_plots = False
```
%% Cell type:markdown id: tags:
## Make the data pycroscopy compatible
Converting the raw data into a pycroscopy compatible hierarchical data format (HDF or .h5) file gives you access to the fast fitting algorithms and powerful analysis functions within pycroscopy
#### H5 files:
* are like smart containers that can store matrices with data, folders to organize these datasets, images, metadata like experimental parameters, links or shortcuts to datasets, etc.
* are readily compatible with high-performance computing facilities
* scale very efficiently from few kilobytes to several terabytes
* can be read and modified using any language including Python, Matlab, C/C++, Java, Fortran, Igor Pro, etc.
#### You can load either of the following:
* Any .mat or .txt parameter file from the original experiment
* A .h5 file generated from the raw data using pycroscopy - skips translation
You can select desired file type by choosing the second option in the pull down menu on the bottom right of the file window
%% Cell type:code id: tags:
``` python
if ui_file_window:
input_file_path = uiGetFile(caption='Select translated .h5 file or raw experiment data',
filter='Parameters for raw G-Line data (*.txt);; \
input_file_path = px.io_utils.uiGetFile(caption='Select translated .h5 file or raw experiment data',
file_filter='Parameters for raw G-Line data (*.txt);; \
Translated file (*.h5)')
else:
input_file_path = '/Volumes/IFgroup/SPM software development/Raw_Data/G_mode/GVS/2015_04_08_PZT_AuCu_nanocaps/GLine_8V_10kHz_256x256_0001/GLine_8V_10kHz_256x256.h5'
folder_path, _ = path.split(input_file_path)
if input_file_path.endswith('.txt'):
print('Translating raw data to h5. Please wait')
tran = px.GLineTranslator()
h5_path = tran.translate(input_file_path)
else:
h5_path = input_file_path
print('Working on:\n' + h5_path)
```
%% Cell type:markdown id: tags:
## Open the .h5 file and extract some basic parameters
%% Cell type:code id: tags:
``` python
hdf = px.ioHDF5(h5_path)
h5_main = px.hdf_utils.getDataSet(hdf.file, 'Raw_Data')[-1]
parms_dict = h5_main.parent.parent.attrs
samp_rate = parms_dict['IO_rate_[Hz]']
ex_freq = parms_dict['BE_center_frequency_[Hz]']
h5_spec_vals = px.hdf_utils.getAuxData(h5_main, auxDataName='Spectroscopic_Values')[0]
pixel_ex_wfm = h5_spec_vals[0, :int(h5_spec_vals.shape[1]/parms_dict['grid_num_cols'])]
```
%% Cell type:markdown id: tags:
##### Inspect the contents of this h5 data file
The file contents are stored in a tree structure, just like files on a conventional computer.
The data is stored as a 2D matrix (position, spectroscopic value) regardless of the dimensionality of the data. Thus, the positions will be arranged as row0-col0, row0-col1.... row0-colN, row1-col0.... and the data for each position is stored as it was chronologically collected
The main dataset is always accompanied by four ancillary datasets that explain the position and spectroscopic value of any given element in the dataset.
Note that G-mode data is acquired line-by-line rather than pixel-by-pixel.
%% Cell type:code id: tags:
``` python
print('Datasets and datagroups within the file:\n------------------------------------')
px.io.hdf_utils.print_tree(hdf.file)
print('\nThe main dataset:\n------------------------------------')
print(h5_main)
print('\nThe ancillary datasets:\n------------------------------------')
print(hdf.file['/Measurement_000/Channel_000/Position_Indices'])
print(hdf.file['/Measurement_000/Channel_000/Position_Values'])
print(hdf.file['/Measurement_000/Channel_000/Spectroscopic_Indices'])
print(hdf.file['/Measurement_000/Channel_000/Spectroscopic_Values'])
print('\nMetadata or attributes in a datagroup\n------------------------------------')
for key in hdf.file['/Measurement_000'].attrs:
print('{} : {}'.format(key, hdf.file['/Measurement_000'].attrs[key]))
```
%% Cell type:markdown id: tags:
## Extract necessary parameters:
%% Cell type:code id: tags:
``` python
parms_dict = h5_main.parent.parent.attrs
samp_rate = parms_dict['IO_rate_[Hz]']
ex_freq = parms_dict['BE_center_frequency_[Hz]']
pixel_ex_wfm = h5_spec_vals[0, :int(h5_spec_vals.shape[1]/parms_dict['grid_num_cols'])]
pts_per_pix = pixel_ex_wfm.size
pts_per_line = h5_main.shape[1]
```
%% Cell type:markdown id: tags:
## Inspect the raw data:
%% Cell type:code id: tags:
``` python
row_ind = 40
raw_row = h5_main[row_ind].reshape(-1, pixel_ex_wfm.size)
raw_row = h5_main[row_ind].reshape(-1, pts_per_pix)
fig, axes = px.plot_utils.plot_loops(pixel_ex_wfm, raw_row, x_label='Bias (V)', title='Raw Measurement',
plots_on_side=4, y_label='Deflection (a.u.)',
subtitles='Row: ' + str(row_ind) + ' Col:')
```
%% Cell type:markdown id: tags:
## Visualizing information in Fourier space
Visualizing in the fourier space provides information about the noise floor, frequencies which are noise dominant or signal dominant, etc.
This visualization will guide the design of signal filters to remove the noise
%% Cell type:code id: tags:
``` python
# Preparing the frequency axis:
w_vec = 1E-3*np.linspace(-0.5*samp_rate, 0.5*samp_rate - samp_rate/pts_per_line, pts_per_line)
row_ind = 40
F_resp = np.fft.fftshift(np.fft.fft(h5_main[row_ind]))
fig, ax = plt.subplots(figsize=(12, 7))
ax.axvline(x=1E-3*ex_freq, color='r', linewidth=2, label='Excitation')
ax.plot(w_vec[int(0.5*len(w_vec)):], np.log10(np.abs(F_resp[int(0.5*len(w_vec)):])), label='Response')
ax.set_xlabel('Frequency (kHz)', fontsize=16)
ax.set_ylabel('Amplitude (a.u.)', fontsize=16)
ax.legend(fontsize=14)
ax.set_xscale('log')
ax.set_xlim(ex_freq*1E-4, samp_rate*0.5E-3)
ax.set_title('Noise Spectrum for row ' + str(row_ind), fontsize=16)
px.plot_utils.set_tick_font_size(ax, 14)
if save_plots:
fig.savefig(os.path.join(other_figures_folder,
'noise_spectrum_line_' + str(row_ind) +'.png'),
format='png', dpi=150);
```
%% Cell type:markdown id: tags:
## Try different FFT filters on the data
Good combinations for frequency filters are:
* Just a HarmonicPassFilter
* LowPassFilter + NoiseBandFilter
It is always a good idea to combine these frequency filters with noise thresholding. Try setting noise tolerance values of 1E-6 to 1E-3/
%% Cell type:code id: tags:
``` python
filter_parms = dict()
filter_parms['noise_threshold'] = 1E-4
filter_parms['comb_[Hz]'] = [ex_freq, 1E+3, 10]
# filter_parms['LPF_cutOff_[Hz]'] = -1
# Noise frequencies - 15.6 kHz ~ 14-17.5, 7.8-8.8, 45-49.9 ~ 48.9414 kHz
# filter_parms['band_filt_[Hz]'] = None # [[8.3E+3, 15.6E+3, 48.9414E+3], [1E+3, 0.5E+3, 0.1E+3]]
# filter_parms['phase_[rad]'] = 0
filter_parms['samp_rate_[Hz]'] = samp_rate
filter_parms['num_pix'] = 1
hpf = px.processing.fft.HarmonicPassFilter(pts_per_line, samp_rate, ex_freq, 1E+3, 10)
lpf = px.processing.fft.LowPassFilter(pts_per_line, samp_rate, 110E+3)
nbf = px.processing.fft.NoiseBandFilter(pts_per_line, samp_rate, [0], [17E+3])
freq_filts = [hpf]
noise_tolerance = 1E-4
# Test filter on a single line:
row_ind = 40
filt_line, fig_filt, axes_filt = px.processing.gmode_utils.test_filter(h5_main[row_ind], filter_parms, samp_rate,
show_plots=True, use_rainbow_plots=False)
fig_filt.savefig(path.join(folder_path, 'FFT_filter_on_line_{}.png'.format(row_ind)), format='png', dpi=300)
filt_line, fig_filt, axes_filt = px.processing.gmode_utils.test_filter(h5_main[row_ind],
frequency_filters=freq_filts,
noise_threshold=noise_tolerance,
show_plots=True)
if save_plots:
fig_filt.savefig(path.join(folder_path, 'FFT_filter_on_line_{}.png'.format(row_ind)), format='png', dpi=300)
filt_row = filt_line.reshape(-1, pixel_ex_wfm.size)
# raw_row = h5_main[row_ind].reshape(-1, pts_per_pix)
fig, axes = px.plot_utils.plot_loops(pixel_ex_wfm, filt_row, x_label='Bias (V)', title='FFT Filtering',
plots_on_side=4, y_label='Deflection (a.u.)',
subtitles='Row: ' + str(row_ind) + ' Col:')
# fig.savefig(path.join(folder_path, 'FFT_filtered_loops_on_line_{}.png'.format(row_ind)), format='png', dpi=300)
if save_plots:
fig.savefig(path.join(folder_path, 'FFT_filtered_loops_on_line_{}.png'.format(row_ind)), format='png', dpi=300)
```
%% Cell type:markdown id: tags:
## Apply selected filter to entire dataset
%% Cell type:code id: tags:
``` python
# h5_filt_grp = px.hdf_utils.findH5group(h5_main, 'FFT_Filtering')[-1]
h5_filt_grp = px.processing.gmode_utils.fft_filter_dataset(h5_main, filter_parms, write_filtered=True)
filter_parms = dict()
if freq_filts is not None:
for filter in freq_filts:
filter_parms.update(filter.get_parms())
if noise_tolerance is not None:
filter_parms['noise_threshold'] = noise_tolerance
h5_filt_grp = px.hdf_utils.check_for_old(h5_main, 'FFT_Filtering', new_parms=filter_parms)
if h5_filt_grp is None:
sig_filt = px.processing.SignalFilter(h5_main, frequency_filters=freq_filts, noise_threshold=noise_tolerance,
write_filtered=True, write_condensed=False, num_pix=1, verbose=True)
h5_filt_grp = sig_filt.compute()
else:
print('Taking previously computed results')
h5_filt = h5_filt_grp['Filtered_Data']
```
%% Cell type:code id: tags:
``` python
# Test to make sure the filter gave the same results
filt_row = h5_filt[row_ind].reshape(-1, pixel_ex_wfm.size)
fig, axes = px.plot_utils.plot_loops(pixel_ex_wfm, filt_row, x_label='Bias (V)', title='FFT Filtering',
plots_on_side=4, y_label='Deflection (a.u.)',
subtitles='Row: ' + str(row_ind) + ' Col:')
```
%% Cell type:markdown id: tags:
## Now break up the filtered lines into "pixels"
Also visualize loops from different pixels
%% Cell type:code id: tags:
``` python
# h5_resh = h5_filt_grp['Filtered_Data-Reshape_000/Reshaped_Data']
h5_resh = px.processing.gmode_utils.reshape_from_lines_to_pixels(h5_filt, pixel_ex_wfm.size, 1)
fig, axes = px.plot_utils.plot_loops(pixel_ex_wfm, h5_resh, x_label='Bias (V)', title='FFT Filtering',
plots_on_side=5, y_label='Deflection (a.u.)')
# fig.savefig(path.join(folder_path, 'FFT_filtered_loops_on_line_{}.png'.format(row_ind)), format='png', dpi=300)
```
%% Cell type:code id: tags:
``` python
hdf.close()
```
%% Cell type:code id: tags:
``` python
```
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment