Commit 832e8d1d authored by Laanait, Nouamane's avatar Laanait, Nouamane

Initial commit

parent 602b3bf6
MIT License
Copyright (c) 2016 Nouamane Laanait.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
include README.md
include LICENSE.txt
\ No newline at end of file
"""
Created on 10/9/17.
@author: Numan Laanait.
email: laanaitn@ornl.gov
"""
from collections import OrderedDict
import json
from stemdl import io_utils
#################################
# templates for network_config #
#################################
layer_keys_list = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5',
'pool1',
'conv6', 'conv7', 'conv8', 'conv9', 'conv10',
'pool2',
'conv11', 'conv12', 'conv13', 'conv14', 'conv15',
'pool3',
'fc_1' ,
'linear_output']
# parameters dictionary
conv_layer_1 = {'type': 'convolutional', 'stride': [1, 1], 'kernel': [6, 6], 'features': 64, 'activation':'relu',
'padding':'SAME'}
conv_layer_2 = {'type': 'convolutional', 'stride': [1, 1], 'kernel': [4, 4], 'features': 128, 'activation':'relu',
'padding':'SAME'}
pool = {'type': 'pooling', 'stride': [2, 2], 'kernel': [2, 2], 'pool_type': 'avg','padding':'SAME'}
conv_layer_3 = {'type': 'convolutional', 'stride': [1, 1], 'kernel': [3, 3], 'features': 256, 'activation':'relu',
'padding':'SAME'}
fully_connected= {'type': 'fully_connected','weights': 1024,'bias': 1024, 'activation': 'tanh','regularize': False}
linear_ouput = {'type': 'linear_output','weights': 3,'bias': 3,'regularize': False}
layer_params_list = [conv_layer_1]*5 + [pool] + [conv_layer_2]*5 + [pool] + [conv_layer_3]*5 + [pool] + \
[fully_connected] + [linear_ouput]
#################################
# templates for hyper-parameters #
#################################
hyper_params_regression = {'network_type': 'regressor', #'network_type': 'classifier'
'optimization': 'ADAM', #'optimization': 'SGD'
'warm_up': True,
'num_epochs_per_decay':30,
'learning_rate_decay_factor': 0.5,
'initial_learning_rate': 1.e-5,
'num_epochs_per_ramp': 10,
'num_epochs_in_warm_up': 100,
'warm_up_max_learning_rate': 1e-3,
'weight_decay': 1.e-4,
'moving_average_decay': 0.9999,
'loss_function': {'type': 'Huber',
'residual_num_epochs_decay': 30,
'residual_initial': 15.,
'residual_minimum': 1.,
'residual_decay_factor': 0.75},
}
if __name__ == '__main__':
io_utils.write_json_network_config('network_regressor.json', layer_params_list, layer_keys_list)
_ = io_utils.load_json_network_config('network_regressor.json')
io_utils.write_json_hyper_params('hyper_params_regressor.json', hyper_params_regression)
_ = io_utils.load_json_hyper_params('hyper_params_regressor.json')
"""
Created on 10/8/17.
@author: Numan Laanait.
email: laanaitn@ornl.gov
"""
from stemdl import inputs
from stemdl import runtime
from stemdl import io_utils
import tensorflow as tf
import argparse
import sys
"""
These FLAGS define variables for a particular TF workflow and are not expected to change.
See runtime.set_flags() for flags that should be set at the CLI.
"""
# Basic parameters describing the training run
tf.app.flags.DEFINE_boolean('log_device_placement', False, """Whether to log device placement.""")
tf.app.flags.DEFINE_boolean('allow_soft_placement', True, """Whether to allow variable soft placement on the device-""" +\
""" multi-gpu runs.""")
tf.app.flags.DEFINE_integer('log_frequency', 10, """How often to log results to the console.""")
tf.app.flags.DEFINE_integer('max_steps', 1000000,"""Number of batches to run.""")
tf.app.flags.DEFINE_integer('num_epochs', 1000000,"""Number of Data Epochs to do training""")
tf.app.flags.DEFINE_integer('NUM_EXAMPLES_PER_EPOCH', 50000,"""Number of classes in training/evaluation data.""")
tf.app.flags.DEFINE_string('worker_name', 'worker', """Name of gpu worker to append to each device ops, scope, etc...""")
# Basic parameters describing the data set.
tf.app.flags.DEFINE_integer('NUM_CLASSES', 3, """Number of classes in training/evaluation data.""")
tf.app.flags.DEFINE_integer('OUTPUT_DIM', 3, """Dimension of the Network's Output""")
tf.app.flags.DEFINE_integer('IMAGE_HEIGHT', 85, """IMAGE HEIGHT""")
tf.app.flags.DEFINE_integer('IMAGE_WIDTH', 120, """IMAGE WIDTH""")
tf.app.flags.DEFINE_integer('IMAGE_DEPTH', 1, """IMAGE DEPTH""")
tf.app.flags.DEFINE_integer('CROP_HEIGHT', 60, """CROP HEIGHT""")
tf.app.flags.DEFINE_integer('CROP_WIDTH', 80, """CROP WIDTH""")
tf.app.flags.DEFINE_boolean('IMAGE_FP16', False, """ Whether to use half-precision format for images.""")
FLAGS = tf.app.flags.FLAGS
def main(argv): # pylint: disable=unused-argument
parser = argparse.ArgumentParser(description='Setup and Run a Convolutional Neural Network')
parser.add_argument('--data_path', type=str, help='path to tfrecords file with images + labels', nargs=1,
required=True)
parser.add_argument('--checkpt_dir', type=str, help='path where to save directory with training data,'+\
'visualization, and TensorBoard events.', nargs=1,
required=True)
parser.add_argument('--network_config', type=str, help='path to .json file with neural net architecture', nargs=1,
required=True)
parser.add_argument('--hyper_params', type=str, help='path to .json file with hyper-parameters', nargs=1,
required=True)
parser.add_argument('--mode', type=str, help="operation mode, must be 'train' or 'eval' ", nargs='*',
default='train')
parser.add_argument('--num_gpus', type=int, help='number of gpus to use during training', nargs='?', default=1)
parser.add_argument('--batch_size', type=int, help='number of images to propagate through the network.'+\
'\nPowers of 2 are processed more efficiently', nargs='?', default=64)
args = parser.parse_args()
# Create the checkpoint directory
if tf.gfile.Exists(args.checkpt_dir[0]):
print('Directory "%s" exists already.\nReloading model from latest checkpoint.' %format(args.checkpt_dir[0]))
else:
tf.gfile.MakeDirs(args.checkpt_dir[0])
# Set additional tf.app.flags
runtime.set_flags(args.checkpt_dir[0], args.batch_size, args.data_path[0])
# load network config file and hyper_parameters
network_config = io_utils.load_json_network_config(args.network_config[0])
hyper_params = io_utils.load_json_hyper_params(args.hyper_params[0])
# train or evaluate
if args.mode == 'train':
runtime.train(network_config, hyper_params, args.data_path[0], tf.app.flags.FLAGS, num_GPUS=args.num_gpus)
if args.mode == 'eval':
pass
if __name__ == '__main__':
main(sys.argv)
from codecs import open
import os
from setuptools import setup, find_packages
here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'README.md')) as f:
long_description = f.read()
requirements = ['numpy>=1.13.0', 'tensorflow>=1.2']
setup(
name='stemdl',
version='0.01',
description='',
long_description=long_description,
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Environment :: Console',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Operating System :: OS Independent',
'Programming Language :: Cython',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: Implementation :: CPython',
'Topic :: Scientific/Engineering :: Chemistry',
'Topic :: Scientific/Engineering :: Physics',
'Topic :: Scientific/Engineering :: Information Analysis',
],
keywords='scientific microscopy data analysis',
packages=find_packages(exclude='tests'),
url='https://code-int.ornl.gov/AtomAI',
license='MIT',
author='N. Laanait',
author_email='laanaitn@ornl.gov',
# I don't remember how to do this correctly!!!. NL
install_requires=requirements,
# package_data={'sample':['dataset_1.dat']}
test_suite='nose.collector',
tests_require='Nose',
dependency='',
dependency_links=[''],
include_package_data=True,
# If there are data files included in your packages that need to be
# installed, specify them here. If using Python 2.6 or less, then these
# have to be included in MANIFEST.in as well.
# package_data={
# 'sample': ['package_data.dat'],
# },
# Although 'package_data' is the preferred approach, in some case you may
# need to place data files outside of your packages. See:
# http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
# In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
# data_files=[('my_data', ['data/data_file'])],
# To provide executable scripts, use entry points in preference to the
# "scripts" keyword. Entry points provide cross-platform support and allow
# pip to create the appropriate form of executable for the target platform.
# entry_points={
# 'console_scripts': [
# 'sample=sample:main',
# ],
# },
)
# Not supporting for __all__
from . import inputs
from .inputs import *
from . import io_utils
from .io_utils import *
from . import network
from .network import *
from . import runtime
from .runtime import *
from .__version__ import version as __version__
from .__version__ import date as __date__
__all__ = ['inputs', 'io_utils', 'network', 'runtime', '__date__', '__version__']
__all__ += inputs.__all__
__all__ += io_utils.__all__
__all__ += network.__all__
__all__ += runtime.__all__
\ No newline at end of file
"""
Created on 10/8/17.
@author: Numan Laanait.
email: laanaitn@ornl.gov
"""
import tensorflow as tf
import numpy as np
class DatasetTFRecords(object):
"""
Handles training and evaluation data operations. \n
Data is read from a TFRecords filename queue.
"""
def __init__(self, filename_queue, flags):
self.filename_queue = filename_queue
self.flags = flags
def decode_image_label(self):
"""
Returns: image, label decoded from tfrecords
"""
reader = tf.TFRecordReader()
key, serialized_example = reader.read(self.filename_queue)
# get raw image bytes and label string
features = tf.parse_single_example(
serialized_example,
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.string),
})
# decode from byte and reshape label and image
label = tf.decode_raw(features['label'], tf.float64)
label.set_shape(self.flags.NUM_CLASSES)
image = tf.decode_raw(features['image_raw'], tf.float16)
image.set_shape([self.flags.IMAGE_HEIGHT * self.flags.IMAGE_WIDTH * self.flags.IMAGE_DEPTH])
image = tf.reshape(image, [self.flags.IMAGE_HEIGHT, self.flags.IMAGE_WIDTH, self.flags.IMAGE_DEPTH])
return image, label
def train_images_labels_batch(self, image_raw, label, distort=False, noise_min=0., noise_max=0.3,
random_glimpses=True, geometric=False):
"""
Returns: batch of images and labels to train on.
"""
if distort:
# we first cast to float32 to do some image operations
image = tf.cast(image_raw, tf.float32)
# Apply image distortions
image = self._distort(image, noise_min, noise_max, geometric=geometric)
else:
image = image_raw
# if running with half-precision we cast to float16
if self.flags.IMAGE_FP16:
image = tf.cast(image, tf.float16)
else:
image = tf.cast(image, tf.float32)
# Generate batch
images, labels = tf.train.shuffle_batch([image, label],
batch_size=self.flags.batch_size,
capacity=100000,
num_threads=16,
min_after_dequeue=10000,
name='shuffle_batch')
# Extract glimpses from training batch
images = self._getGlimpses(images, random=random_glimpses)
# Display the training images in the Tensorboard visualizer.
tf.summary.image('Train_Images', images, max_outputs=1)
# change to NCHW format
images = tf.transpose(images, perm=[0, 3, 1, 2])
return images, labels
def eval_images_labels_batch(self, image_raw, label, distort= False, noise_min=0., noise_max=0.3,
random_glimpses=False, geometric=False):
"""
Returns: batch of images and labels to test on.
"""
if distort:
# we first cast to float32 to do some image operations
image = tf.cast(image_raw, tf.float32)
# Apply image distortions
image = self._distort(image, noise_min, noise_max, geometric=geometric)
else:
image = image_raw
# if running with half-precision we cast to float16
if self.flags.IMAGE_FP16:
image = tf.cast(image, tf.float16)
else:
image = tf.cast(image, tf.float32)
# Generate batch
images, labels = tf.train.shuffle_batch([image, label],
batch_size=self.flags.batch_size,
capacity=5100,
num_threads=4,
min_after_dequeue=100,
name='shuffle_batch')
#extract glimpses from evaluation batch
images = self._getGlimpses(images, random=random_glimpses)
# Display the training images in the visualizer.
tf.summary.image('Test_Images', images, max_outputs=1)
return images, labels
@staticmethod
def _distort(image, noise_min, noise_max, geometric=False):
"""
Performs distortions on an image: noise + global affine transformations.
Args:
image: 3D Tensor
noise_min:float, lower limit in (0,1)
noise_max:float, upper limit in (0,1)
geometric: bool, apply affine distortion
Returns:
distorted_image: 3D Tensor
"""
# Apply random global affine transformations
if geometric:
# Setting bounds and generating random values for scaling and rotations
scale_X = np.random.normal(1.0, 0.05, size=1)
scale_Y = np.random.normal(1.0, 0.05, size=1)
theta_angle = np.random.normal(0., 0.2, size=1)
nu_angle = np.random.normal(0., 0.2, size=1)
# Constructing transfomation matrix
a_0 = scale_X * np.cos(np.deg2rad(theta_angle))
a_1 = -scale_Y * np.sin(np.deg2rad(theta_angle + nu_angle))
a_2 = 0.
b_0 = scale_X * np.sin(np.deg2rad(theta_angle))
b_1 = scale_Y * np.cos(np.deg2rad(theta_angle + nu_angle))
b_2 = 0.
c_0 = 0.
c_1 = 0.
affine_transform = tf.constant(np.array([a_0, a_1, a_2, b_0, b_1, b_2, c_0, c_1]).flatten(),
dtype=tf.float32)
# Transform
image = tf.contrib.image.transform(image, affine_transform,
interpolation='BILINEAR')
# Apply noise
alpha = tf.random_uniform([1], minval=noise_min, maxval=noise_max)
noise = tf.random_uniform(image.shape, dtype=tf.float32)
image = (1 - alpha[0]) * image + alpha[0] * noise
return image
def _getGlimpses(self, batch_images, **kwargs):
"""
Get bounded glimpses from images, corresponding to ~ 2x1 supercell
:param batch_images: batch of training images
:return: batch of glimpses
"""
# set size of glimpses
y_size, x_size = self.flags.IMAGE_HEIGHT, self.flags.IMAGE_WIDTH
crop_y_size, crop_x_size = self.flags.CROP_HEIGHT, self.flags.CROP_WIDTH
size = tf.constant(value=[crop_y_size, crop_x_size],
dtype=tf.int32)
random = kwargs.get('random', False)
if random is 'uniform':
# generate uniform random window centers for the batch with overlap with input
y_low, y_high = int(crop_y_size / 2), int(y_size - crop_y_size / 2)
x_low, x_high = int(crop_x_size / 2), int(x_size - crop_x_size / 2)
cen_y = tf.random_uniform([self.flags.batch_size], minval=y_low, maxval=y_high)
cen_x = tf.random_uniform([self.flags.batch_size], minval=x_low, maxval=x_high)
offsets = tf.stack([cen_y, cen_x], axis=1)
if random is 'normal':
# generate normal random window centers for the batch with overlap with input
cen_y = tf.random_normal([self.flags.batch_size], mean=y_size / 2, stddev=4.)
cen_x = tf.random_normal([self.flags.batch_size], mean=x_size / 2, stddev=4.)
offsets = tf.stack([cen_y, cen_x], axis=1)
if not random:
# fixed crop
cen_y = np.ones((self.flags.batch_size,), dtype=np.int32) * 38
cen_x = np.ones((self.flags.batch_size,), dtype=np.int32) * 70
offsets = np.vstack([cen_y, cen_x]).T
offsets = tf.constant(value=offsets, dtype=tf.float32)
# extract glimpses
glimpse_batch = tf.image.extract_glimpse(batch_images, size, offsets, centered=False,
normalized=False,
uniform_noise=False,
name='batch_glimpses')
return glimpse_batch
\ No newline at end of file
"""
Created on 10/8/17.
@author: Numan Laanait.
email: laanaitn@ornl.gov
"""
from collections import OrderedDict
import json
# JSON utility functions
def write_json_network_config(file, layer_keys, layer_params):
"""
Constructs and OrderedDict object and writes it .json file
:param file: string, path to file
:param layer_keys: list(string), names of layers
:param layer_params: list(dict), dictionary of layer parameters
:return: None
"""
assert len(layer_keys) == len(layer_params), '# of layer names and # of layer parameter dictionaries do not match!'
network_config = OrderedDict(zip(layer_keys_list, layer_params_list))
with open(file, mode='w') as f:
json.dump(network_config, f, indent="\t")
print('Wrote %d NN layers to %s' % (len(network_config.keys()), file))
def load_json_network_config(file):
"""
Reads a .json file and returns and OrderedDict object to be used to construct neural nets.
:param file: .json file.
:return: network_config OrderedDict
"""
with open(file, mode='r') as f:
def _as_ordered_dict(val):
return OrderedDict(val)
def _as_list(val):
return list(val)
output = json.load(f, object_hook=_as_ordered_dict, object_pairs_hook=_as_ordered_dict)
network_config = OrderedDict(output)
print('Read %d NN layers from %s' % (len(network_config.keys()), file))
return network_config
def write_json_hyper_params(file, hyper_params):
"""
Write hyper_parameters to .json file
:param file: string, path to .json file
:param hyper_params: dict, hyper-paramters.
:return: None
"""
with open(file, mode='w') as f:
json.dump(hyper_params, f, indent="\t")
print('Wrote %d hyperparameters to %s' % (len(hyper_params.keys()), file))
def load_json_hyper_params(file):
"""
Loads hyper_parameters dictionary from .json file
:param file: string, path to .json file
:return: dict, hyper-paramters.
"""
with open(file, mode='r') as f:
hyper_params = json.load(f)
print('Read %d hyperparameters from %s' % (len(hyper_params.keys()), file))
return hyper_params
This diff is collapsed.
This diff is collapsed.
[bdist_wheel]
# This flag says that the code is written to work on both Python 2 and Python
# 3. If at all possible, it is good practice to do this. If you cannot, you
# will need to generate wheels for each Python version that you support.
#universal=1
\ No newline at end of file
"""
Created on 10/9/17.
@author: Numan Laanait.
email: laanaitn@ornl.gov
"""
from codecs import open
import os
on_rtd = os.environ.get('READTHEDOCS') == 'True'
from setuptools import setup, find_packages
here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'README.rst')) as f:
long_description = f.read()
if on_rtd:
requirements = ['psutil', 'xlrd>=1.0.0']
else:
requirements = ['numpy_groupies>=0.9.6', 'pyqtgraph>=0.10',
'h5py>=2.6.0', 'igor', 'matplotlib>=2.0.0',
'scikit-learn>=0.17.1', 'xlrd>=1.0.0',
'psutil', 'scikit-image>=0.12.3', 'scipy>=0.17.1',
'numpy>=1.11.0', 'ipywidgets>=5.2.2', 'ipython>=5.1.0']
setup(
name='pycroscopy',
version='0.0a51',
description='A suite of Python libraries for high performance scientific computing of microscopy data.',
long_description=long_description,
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Environment :: Console',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Operating System :: OS Independent',
'Programming Language :: Cython',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: Implementation :: CPython',
'Topic :: Scientific/Engineering :: Chemistry',
'Topic :: Scientific/Engineering :: Physics',
'Topic :: Scientific/Engineering :: Information Analysis',
],
keywords='scientific microscopy data analysis',
packages=find_packages(exclude='tests'),
url='http://github.com/pycroscopy/pycroscopy',
license='MIT',
author='S. Somnath, C. R. Smith, N. Laanait',
author_email='pycroscopy@gmail.com',
# I don't remember how to do this correctly!!!. NL
install_requires=requirements,
# package_data={'sample':['dataset_1.dat']}
test_suite='nose.collector',
tests_require='Nose',
dependency='',
dependency_links=[''],
include_package_data=True,
# If there are data files included in your packages that need to be
# installed, specify them here. If using Python 2.6 or less, then these
# have to be included in MANIFEST.in as well.
# package_data={
# 'sample': ['package_data.dat'],
# },
# Although 'package_data' is the preferred approach, in some case you may
# need to place data files outside of your packages. See:
# http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
# In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
# data_files=[('my_data', ['data/data_file'])],
# To provide executable scripts, use entry points in preference to the
# "scripts" keyword. Entry points provide cross-platform support and allow
# pip to create the appropriate form of executable for the target platform.
# entry_points={
# 'console_scripts': [
# 'sample=sample:main',
# ],
# },
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment