Commit d0a0d21d authored by Nouamane Laanait's avatar Nouamane Laanait
Browse files

summit script files

parent 1080dcc7

scripts/summit_scripts/h5_combine.py

0 → 100644
+42 −0
Original line number Diff line number Diff line 
#from namsa import SupercellBuilder, MSAGPU

#from utils import *

import numpy as np

from time import time

import sys, os, re

import h5py

#from mpi4py import MPI

#from itertools import chain

#import tensorflow as tf

#import lmdb



#comm = MPI.COMM_WORLD

#comm_size = comm.Get_size()

#comm_rank = comm.Get_rank()





def main(h5_trg, h5_dir):

    h5_paths = [os.path.join(h5_dir, itm) for itm in os.listdir(h5_dir)] 

    #if os.path.exists(h5path):

    #    mode ='r+'

    #else:

    #    mode ='w'

    mode = 'w'

    i = 0

    with h5py.File(h5_trg, mode=mode) as f_trg:

        for h5_path_src in h5_paths:

            try:

                with h5py.File(h5_path_src, mode='r') as f_src:

                    for (_, g_trg) in f_src.items():

                        #print([itm for _, itm in g_trg.attrs.items()])

                        f_src.copy(g_trg,f_trg['/'], name='sample_%d'%i)

                        i +=1

                print('copied contents of %s' % h5_path_src)

            except Exception as e:

                print('skipped %s, error=%s' % (h5_path_src, format(e)))



if __name__ == "__main__":

    if len(sys.argv) > 2:

        h5_trg, h5_dir = sys.argv[-2:]

        main(h5_trg, h5_dir)

    else:

        print("Pass directory paths for sim input files and h5 output files")

scripts/summit_scripts/h5comb_job.bs

0 → 100755
+46 −0
Original line number Diff line number Diff line 
#!/bin/bash -l

#BSUB -P LRN001 

#BSUB -J namsa 

#BSUB -o logs.o%J

#BSUB -W 02:00 

#BSUB -nnodes 1 

#BSUB -alloc_flags "smt4 nvme maximizegpfs"

#BSUB -q batch

##BSUB -N

##BSUB -csm y

##BSUB -alloc_flags "smt4 gpumps nvme"

NODES=$(cat ${LSB_DJOB_HOSTFILE} | sort | uniq | grep -v login | grep -v batch | wc -l)

BUILDS=${PROJWORK}/lrn001/nl/builds



### modules ###

module unload darshan-runtime

module load gcc/6.4.0

module load fftw hdf5 cuda



### python ###

PYTHON=${BUILDS}/miniconda3

export PATH=$PYTHON/bin:$PATH 

export PYTHONIOENCODING="utf8"

export LD_LIBRARY_PATH=${PYTHON}/lib:$LD_LIBRARY_PATH

#CONDA_ENV_NAME="torch1p0"

CONDA_ENV_NAME="tf1p12"

source activate $CONDA_ENV_NAME

echo $(which python)



LOG="$(pwd)/output_h5comb.log"



H5DIR="$(pwd)/data/h5_params"

#SAVEDIR="$(pwd)/data/h5_files"

H5TRG="$(pwd)/data/master_params_0401.h5"

#SAVEDIR="$(pwd)/data/tfrecords"

#SAVEDIR="/mnt/bb/${USER}"

#SAVEMODE="tfrecord"

#SAVEMODE="h5"

#export PYCUDA_DISABLE_CACHE=1

EXEC="python -W ignore -u h5_combine.py $H5TRG $H5DIR"

#python -W ignore -u h5_combine.py $H5TRG $H5DIR

   

### run ###

jsrun -n${NODES} -a1 -c42 -g0 -r1 --bind=none --launch_distribution=cyclic ${EXEC} > $LOG 

#jsrun -n${NODES} -a1 -c42 -g0 -r1 ${EXEC} > $LOG

#--bind=proportional-packed:7 --launch_distribution=packed ${EXEC} > $LOG

scripts/summit_scripts/h5comb_job.sh

0 → 100755
+46 −0
Original line number Diff line number Diff line 
#!/bin/bash -l

#BSUB -P LRN001 

#BSUB -J namsa 

#BSUB -o logs.o%J

#BSUB -W 02:00 

#BSUB -nnodes 1 

#BSUB -alloc_flags "smt4 nvme maximizegpfs"

#BSUB -q batch

##BSUB -N

##BSUB -csm y

##BSUB -alloc_flags "smt4 gpumps nvme"

NODES=$(cat ${LSB_DJOB_HOSTFILE} | sort | uniq | grep -v login | grep -v batch | wc -l)

BUILDS=${PROJWORK}/lrn001/nl/builds



### modules ###

module unload darshan-runtime

module load gcc/6.4.0

module load fftw hdf5 cuda



### python ###

PYTHON=${BUILDS}/miniconda3

export PATH=$PYTHON/bin:$PATH 

export PYTHONIOENCODING="utf8"

export LD_LIBRARY_PATH=${PYTHON}/lib:$LD_LIBRARY_PATH

#CONDA_ENV_NAME="torch1p0"

CONDA_ENV_NAME="tf1p12"

source activate $CONDA_ENV_NAME

echo $(which python)



LOG="$(pwd)/output_h5comb.log"



H5DIR="$(pwd)/data/h5_params"

#SAVEDIR="$(pwd)/data/h5_files"

H5TRG="$(pwd)/data/master_params_0331.h5"

#SAVEDIR="$(pwd)/data/tfrecords"

#SAVEDIR="/mnt/bb/${USER}"

#SAVEMODE="tfrecord"

#SAVEMODE="h5"

#export PYCUDA_DISABLE_CACHE=1

EXEC="python -W ignore -u h5_combine.py $H5TRG $H5DIR"

#python -W ignore -u h5_combine.py $H5TRG $H5DIR

   

### run ###

jsrun -n${NODES} -a1 -c42 -g0 -r1 --bind=none --launch_distribution=cyclic ${EXEC} > $LOG 

#jsrun -n${NODES} -a1 -c42 -g0 -r1 ${EXEC} > $LOG

#--bind=proportional-packed:7 --launch_distribution=packed ${EXEC} > $LOG

scripts/summit_scripts/lmdb_export.py

0 → 100644
+37 −0
Original line number Diff line number Diff line 
import lmdb

import numpy as np

import sys, os, subprocess, shlex



def read_lmdb(save_dir, lmdb_path, index=None):

    env = lmdb.open(lmdb_path, create=False, readahead=False, readonly=True, writemap=False, lock=False)

    num_samples = (env.stat()['entries'] - 4)//2 ## TODO: remove hard-coded # of headers by storing #samples key, val

    first_record = 0

    if index == 'random':

        index = [np.random.randint(first_record, num_samples)]

    else:

        index = np.arange(first_record, num_samples)

    data_specs={'label_shape': [512,512], 'image_shape': [1024, 512, 512],

            'label_dtype':'float16', 'image_dtype': 'float16', 'label_key':'potential_', 'image_key': 'cbed_'}

    for idx in index:

        image_key = bytes(data_specs['image_key']+str(idx), "ascii")

        label_key = bytes(data_specs['label_key']+str(idx), "ascii")

        with env.begin(write=False, buffers=True) as txn:

            image_bytes = txn.get(image_key)

            label_bytes = txn.get(label_key)

            label = np.frombuffer(label_bytes, dtype=data_specs['label_dtype'])

            label = label.reshape(data_specs['label_shape'])

            image = np.frombuffer(image_bytes, dtype=data_specs['image_dtype'])

            image = image.reshape(data_specs['image_shape'])

        np.save(os.path.join(save_dir,'cbed_%d.npy'%idx), image)

        np.save(os.path.join(save_dir,'pot_%d.npy'%idx), label)

    return

if __name__ == "__main__":

    lmdb_path = sys.argv[-2]

    save_dir = sys.argv[-1]

    read_lmdb(save_dir, lmdb_path)

    tar_args = "tar -czvf %s %s" %(os.path.join(os.getcwd(),"saved_arr.tar"), save_dir)

    tar_args = shlex.split(tar_args)

    try:

        subprocess.run(tar_args, check=True)

    except subprocess.SubprocessError as e:

        print("%s" % format(e))

scripts/summit_scripts/namsa_install.sh

+7 −4
Original line number Diff line number Diff line 
#!/bin/bash -l

HOME="/gpfs/wolf/gen113/scratch/nl7/work"

PYTHON=${HOME}/anaconda3 

cd ${HOME}/MSA/namsa

${PYTHON}/bin/python -u setup.py install

BUILDS=${PROJWORK}/lrn001/nl/builds

PYTHON=${BUILDS}/miniconda3

export PATH=$PYTHON/bin:$PATH

#conda install --name "tf1p12" matplotlib

source activate "tf1p12" 

cd namsa

python -u setup.py install