Commit de542459 authored by Nichols, Stephen's avatar Nichols, Stephen
Browse files

Making the jump to Frontier

parent baf456c9

Makefile

0 → 100644
+9 −0
Original line number Diff line number Diff line 
SHELL=/bin/bash



clean:

	rm -f *.o *.mod *.x *.s



realclean:

	rm -f *.o *.mod *.x *.s *~

Makefile_frontier

0 → 100644
+134 −0
Original line number Diff line number Diff line 
SHELL=/bin/bash



#MACHINE = Summit

MACHINE = Frontier



COMPILER = Native

#COMPILER = GCC



#CPU_or_GPU = CPU

CPU_or_GPU = GPU



#### Summit

ifeq ($(MACHINE),Summit)

   ifeq ($(COMPILER),Native)

      ifeq ($(CPU_or_GPU),CPU)

         ## CPU-Only 

         ## enable OpenMP without defining USE_OPENMP_OFFLOAD to use the omp_get_wtime 

         ## function and this makes me use the xlf_r compiler instead of xlf

         FC      = xlf_r

         FFLAGS = -O3 -qsmp=omp -qfree -qpreprocess -qtune=pwr9 -qarch=pwr9 -W@,"-v"

         LDFLAGS = -qsmp=omp

      else

         ## GPUs with OpenMP Offloading

         FC      = xlcuf

         FFLAGS = -O3 -qfree -qpreprocess -qsmp=omp -qoffload -qcuda -qtune=pwr9 -qarch=pwr9 -qtgtarch=sm_70 -W@,"-v"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_CUDA

         LDFLAGS = -qsmp=omp -qoffload -L${OLCF_CUDA_ROOT}/lib64 -lnvToolsExt -lcudart

      endif

   endif



   ifeq ($(COMPILER),GCC)

      ifeq ($(CPU_or_GPU),GPU)

         FC      = gfortran

         FFLAGS = -O3 -ffree-form -cpp -fopenmp -foffload="-lm -latomic"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_CUDA

         LDFLAGS = -fopenmp -foffload="-lm -latomic" -L${OLCF_CUDA_ROOT}/lib64 -lnvToolsExt -lcudart

      else

         FC      = gfortran

         FFLAGS = -O3 -ffree-form -cpp

         LDFLAGS =

      endif

   endif

endif



#### Frontier (GCC option may not be available)

ifeq ($(MACHINE),Frontier)

   ROCM_LIB_PATH=${OLCF_ROCM_ROOT}/lib

   ROCM_INC_PATH=${OLCF_ROCM_ROOT}/include



   ifeq ($(COMPILER),Native)

      FC = ftn



      ifeq ($(CPU_or_GPU),GPU)

         ### GPU

         FFLAGS = -N1023 -O3 -homp,thread2 -em -ef -eT -ffree -DUSE_OPENMP_OFFLOAD -DUSE_AMD 

         ##FFLAGS += -I$(ROCM_INC_PATH)

         #FFLAGS += -eo

         #FFLAGS += -hnofma

         #FFLAGS += -hflex_mp=strict -hfp2=noapprox -hacc_model=no_fast_addr -hacc_model=auto_async_none

         LDFLAGS = -homp,thread2 -L$(ROCM_LIB_PATH) -lamdhip64 -lhsa-runtime64

      else

         ### CPU-only without OpenMP

         #FFLAGS = -N1023 -O3 -hnoomp -em -ef -eT -ffree

         #LDFLAGS = -hnoomp



         ### CPU-only with OpenMP flag ... no cpu-based openmp is in the code, but I 

         ### want to use omp_get_wtime function ... don't define USE_OPENMP_OFFLOAD

         FFLAGS = -N1023 -O3 -homp -em -ef -eT -ffree

         LDFLAGS = -homp

      endif

   endif



   ifeq ($(COMPILER),GCC)

      ifeq ($(CPU_or_GPU),GPU)

         FC = gfortran

         FFLAGS = -Ofast -ffree-form -cpp -fopenmp -foffload="-march=gfx908 -lm"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_AMD

         LDFLAGS = -fopenmp -foffload="-march=gfx908 -lm"

         #LDFLAGS += -L$(ROCM_LIB_PATH) -lamdhip64 -lhsa-runtime64

         LDFLAGS += -L$(ROCM_LIB_PATH) -lamdhip64

      else

         ### CPU-only without OpenMP ... uses gfortran timer

         #FC = gfortran

         #FFLAGS = -O3 -ffree-form -ffree-line-length-none -cpp

         #LDFLAGS =



         ### CPU-only with OpenMP flag ... no cpu-based openmp is in the code, but I 

         ### want to use omp_get_wtime function ... don't define USE_OPENMP_OFFLOAD

         FC = gfortran

         FFLAGS = -O3 -fopenmp -ffree-form -ffree-line-length-none -cpp

         LDFLAGS = -fopenmp

      endif

   endif

endif



F_SRC = custom_linsolve.f90



MODS =



ifneq (,$(findstring USE_CUDA,$(FFLAGS)))

   MODS += my_cudainterfaces.f90

   MODS += nvtx.f90

endif

ifneq (,$(findstring USE_AMD,$(FFLAGS)))

   MODS += my_hipinterfaces.f90

endif



MODS += global_parameters.f90 timing_functions.f90 matrix_ops.f90



OBJECTS = $(MODS:.f90=.o) $(F_SRC:.f90=.o)

EXE = custom_linsolve.x



# rules for makefile

#$(FC) -o $@ $^ $(LDFLAGS)

$(EXE): $(OBJECTS)

	$(FC) $(LDFLAGS) -o $@ $^



%.o:%.f90

	$(FC) $(FFLAGS) -c -o $@ $<



%.o:%.f95

	$(FC) $(FFLAGS) -c -o $@ $<



default: all



all: $(EXE)



clean:

	rm -f *.o *.mod *.x *.s



clobber:

	rm -f *.o *.mod *.x *.s *~

Makefile_summit

+86 −29
Original line number Diff line number Diff line 
SHELL=/bin/bash



MACHINE = Summit

### the Spock compiler flags also work on Crusher

#MACHINE = Spock

#MACHINE = Frontier



COMPILER = Native

#COMPILER = GCC



#CPU_or_GPU = CPU

CPU_or_GPU = GPU



#### Summit

ifeq ($(MACHINE),Summit)

   ifeq ($(COMPILER),Native)

      ifeq ($(CPU_or_GPU),CPU)

         ## CPU-Only 

   #FC      = xlf

   #FFLAGS = -O3 -qfree -qpreprocess -qtune=pwr9 -qarch=pwr9 -qtgtarch=sm_70 -W@,"-v"

   #LDFLAGS =



         ## enable OpenMP without defining USE_OPENMP_OFFLOAD to use the omp_get_wtime 

         ## function and this makes me use the xlf_r compiler instead of xlf

         FC      = xlf_r

         FFLAGS = -O3 -qsmp=omp -qfree -qpreprocess -qtune=pwr9 -qarch=pwr9 -W@,"-v"

         LDFLAGS = -qsmp=omp

      else

         ## GPUs with OpenMP Offloading

         FC      = xlcuf

         FFLAGS = -O3 -qfree -qpreprocess -qsmp=omp -qoffload -qcuda -qtune=pwr9 -qarch=pwr9 -qtgtarch=sm_70 -W@,"-v"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_CUDA

         LDFLAGS = -qsmp=omp -qoffload -L${OLCF_CUDA_ROOT}/lib64 -lnvToolsExt -lcudart

      endif

   endif



   ifeq ($(COMPILER),GCC)

      ifeq ($(CPU_or_GPU),GPU)

         FC      = gfortran

         FFLAGS = -O3 -ffree-form -cpp -fopenmp -foffload="-lm -latomic"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_CUDA

         LDFLAGS = -fopenmp -foffload="-lm -latomic" -L${OLCF_CUDA_ROOT}/lib64 -lnvToolsExt -lcudart

      else

         FC      = gfortran

         FFLAGS = -O3 -ffree-form -cpp

         LDFLAGS =

      endif

   endif

endif



#### Spock and Crusher

ifeq ($(MACHINE),Spock)

#### Spock and Crusher (GCC option not available on Crusher yet)

ifeq ($(MACHINE),Crusher)

   ROCM_LIB_PATH=${OLCF_ROCM_ROOT}/lib

   ROCM_INC_PATH=${OLCF_ROCM_ROOT}/include



   ifeq ($(COMPILER),Native)

      FC = ftn



      ifeq ($(CPU_or_GPU),GPU)

         ### GPU

   FFLAGS = -N1023 -O3 -homp -em -ef -eT -ffree -DUSE_OPENMP_OFFLOAD -DUSE_AMD 

         FFLAGS = -N1023 -O3 -homp -eo -em -ef -eT -ffree -DUSE_OPENMP_OFFLOAD -DUSE_AMD 

         ##FFLAGS += -I$(ROCM_INC_PATH)

         #FFLAGS += -hnofma

         #FFLAGS += -hflex_mp=strict -hfp2=noapprox -hacc_model=no_fast_addr -hacc_model=auto_async_none

         LDFLAGS = -homp -L$(ROCM_LIB_PATH) -lamdhip64 -lhsa-runtime64



      else

         ### CPU-only without OpenMP

         #FFLAGS = -N1023 -O3 -hnoomp -em -ef -eT -ffree

         #LDFLAGS = -hnoomp



   ### CPU-only with OpenMP flag ... no cpu-based openmp is in the code

   #FFLAGS = -N1023 -O3 -homp -em -ef -eT -ffree

   #LDFLAGS = -homp

         ### CPU-only with OpenMP flag ... no cpu-based openmp is in the code, but I 

         ### want to use omp_get_wtime function ... don't define USE_OPENMP_OFFLOAD

         FFLAGS = -N1023 -O3 -homp -em -ef -eT -ffree

         LDFLAGS = -homp

      endif

   endif



   ifeq ($(COMPILER),GCC)

      ifeq ($(CPU_or_GPU),GPU)

         FC = gfortran

         FFLAGS = -Ofast -ffree-form -cpp -fopenmp -foffload="-march=gfx908 -lm"

         FFLAGS += -DUSE_OPENMP_OFFLOAD -DUSE_AMD

         LDFLAGS = -fopenmp -foffload="-march=gfx908 -lm"

         #LDFLAGS += -L$(ROCM_LIB_PATH) -lamdhip64 -lhsa-runtime64

         LDFLAGS += -L$(ROCM_LIB_PATH) -lamdhip64

      else

         ### CPU-only without OpenMP ... uses gfortran timer

         #FC = gfortran

         #FFLAGS = -O3 -ffree-form -ffree-line-length-none -cpp

         #LDFLAGS =



         ### CPU-only with OpenMP flag ... no cpu-based openmp is in the code, but I 

         ### want to use omp_get_wtime function ... don't define USE_OPENMP_OFFLOAD

         FC = gfortran

         FFLAGS = -O3 -fopenmp -ffree-form -ffree-line-length-none -cpp

         LDFLAGS = -fopenmp

      endif

   endif

endif



F_SRC = custom_linsolve.f90
@@ -57,12 +110,16 @@ OBJECTS = $(MODS:.f90=.o) $(F_SRC:.f90=.o) 
EXE = custom_linsolve.x



# rules for makefile

#$(FC) -o $@ $^ $(LDFLAGS)

$(EXE): $(OBJECTS)

	$(FC) -o $@ $^ $(LDFLAGS)

	$(FC) $(LDFLAGS) -o $@ $^



%.o:%.f90

	$(FC) $(FFLAGS) -c -o $@ $<



%.o:%.f95

	$(FC) $(FFLAGS) -c -o $@ $<



default: all



all: $(EXE)

SCRIPTS/build_frontier.sh

0 → 100755
+13 −0
Original line number Diff line number Diff line 
#!/bin/bash



source setUpModules_frontier.sh



export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}



module list



make -f Makefile_frontier clean

make -f Makefile_frontier



# check the executable in this environment

ldd custom_linsolve.x

SCRIPTS/setUpModules_frontier.sh

0 → 100755
+7 −0
Original line number Diff line number Diff line 
#!/bin/bash



### rocm and gpu modules

module load cpe/23.09

module load rocm

module load craype-accel-amd-gfx90a