Commit 02ee007d authored by Nouamane Laanait's avatar Nouamane Laanait
Browse files

small tweaks- most important is moding out comm_rank by 6 to get gpu id

parent 00b750db

scripts/summit_scripts/job.bs

+3 −2
Original line number Diff line number Diff line
@@ -3,9 +3,10 @@ 
#BSUB -J namsa 

#BSUB -o logs.o%J

#BSUB -W 15

#BSUB -nnodes 1 

#BSUB -nnodes 12 

#BSUB -alloc_flags "smt4 nvme"

#BSUB -q batch

##BSUB -N

##BSUB -csm y

##BSUB -alloc_flags "smt4 gpumps nvme"
@@ -28,7 +29,7 @@ export CIF="${IODIR}/cif_files/Si.cif" 
export H5F="${IODIR}/outputs_${LSB_JOBID}.h5"

LOG="${IODIR}/namsa_log_${LSB_JOBID}.log"



EXEC="${PYTHON}/bin/python -u ./test_namsa_mpi.py 1.0 0"

EXEC="${PYTHON}/bin/python -u ./test_namsa_mpi.py 0.75 0"

   

### run ###

jsrun -n${NODES} -a6 -c42 -g6 -r1 --bind=proportional-packed:7 --launch_distribution=packed stdbuf -o0 ./launch.sh "${EXEC}" > $LOG

scripts/summit_scripts/test_namsa_mpi.py

+15 −10
Original line number Diff line number Diff line
@@ -11,15 +11,18 @@ comm = MPI.COMM_WORLD 
comm_size = comm.Get_size()

comm_rank = comm.Get_rank()



#job_pid = os.environ.get('LS_JOBPID')

#job_id = os.environ.get('LSB_JOBID')

#hosts = np.array(os.environ.get('LSB_HOSTS').split(' '))

#job_hosts = np.unique(hosts)





if comm_rank == 0:

    try: 

        job_pid = os.environ.get('LS_JOBPID')

        job_id = os.environ.get('LSB_JOBID')

        hosts = np.array(os.environ.get('LSB_HOSTS').split(' '))

        job_hosts = np.unique(hosts)





if comm_rank == 0:

    try: 

        print('JOB_ID:%s, JOB_PID:%s, HOSTS:%s' %(job_id, job_pid, format(job_hosts)))

    except:

        print('NO JOB VARIABLES!!!')
@@ -46,7 +49,7 @@ def run(h5_file= None, step=2.5, gpu_rank=0): 
    msa.calc_atomic_potentials()

    slice_thickness = 3.135 #

    msa.build_potential_slices(slice_thickness)

    print('rank %d: time to build atomic potential: %2.2f' % (gpu_rank, time() - t))

    print('rank %d: time to build atomic potential: %2.2f' % (comm_rank, time() - t))

    aberration_params = {'C1':500., 'C3': 3.3e7, 'C5':44e7}

    probe_params = {'smooth_apert': True, 'scherzer': False, 'apert_smooth': 60, 

                'aberration_dict':aberration_params, 'spherical_phase': True}
@@ -59,10 +62,12 @@ def run(h5_file= None, step=2.5, gpu_rank=0): 
        msa.multislice(h5_write=h5_file)

    else:

        msa.multislice()

    print('rank %d: time to propagate probes:%2.2f' % (gpu_rank, time() - t))

    print('rank %02d: time to propagate probes:%2.2f' % (comm_rank, time() - t))

    if comm_rank == 0: print('Simulation Finished Successfully!')

    return

    

if __name__ == '__main__':

    gpu_rank = int(np.mod(comm_rank,6))

    if len(sys.argv) == 2:

        step = float(sys.argv[-1])

    if len(sys.argv) == 3:
@@ -71,9 +76,9 @@ if __name__ == '__main__': 
        if write:

            with h5py.File(h5_path, driver='mpio', mode='w', comm=MPI.COMM_WORLD, libver='latest') as f:

                f.atomic = False

                run(h5_file=f, step=step, gpu_rank=comm_rank)

                run(h5_file=f, step=step, gpu_rank=gpu_rank)

        else:

            run(step=step, gpu_rank=comm_rank)

            run(step=step, gpu_rank=gpu_rank)

    else:

        run(step=2.5, gpu_rank=comm_rank)

        run(step=2.5, gpu_rank=gpu_rank)