Commit d32f26ae authored by Papatheodore, Thomas's avatar Papatheodore, Thomas
Browse files

added GPU-mapping script and changed Makefiles

parent 0ae01c3e

Makefile.CC-cmpi

0 → 100644
+16 −0
Original line number Diff line number Diff line 
COMP  = CC

FLAGS = -std=c++11 -fopenmp -D__HIP_PLATFORM_HCC__ -D__HIP_ROCclr__ -D__HIP_ARCH_GFX908__=1



INCLUDES  = -I$(HIP_PATH)/include

LIBRARIES = -L$(HIP_PATH)/lib -lamdhip64



hello_jobstep: hello_jobstep.o

	$(COMP) $(FLAGS) $(LIBRARIES) hello_jobstep.o -o hello_jobstep



hello_jobstep.o: hello_jobstep.cpp

	$(COMP) $(FLAGS) $(INCLUDES) -c hello_jobstep.cpp



.PHONY: clean



clean:

	rm -f hello_jobstep *.o

gpu_map.sh

0 → 100755
+4 −0
Original line number Diff line number Diff line 
#!/bin/bash



export ROCR_VISIBLE_DEVICES=$(($SLURM_LOCALID % 4))

exec $*

hello_jobstep.cpp

+9 −14
Original line number Diff line number Diff line
@@ -41,21 +41,16 @@ int main(int argc, char *argv[]){ 
	int resultlength;

	MPI_Get_processor_name(name, &resultlength);



    const char* gpu_id_list = getenv("HIP_VISIBLE_DEVICES");

    // Find how many GPUs are set in environment variable

    const char* gpu_id_list = getenv("ROCR_VISIBLE_DEVICES");



	// Find how many GPUs HIP runtime says are available

	int num_devices = 0;

    hipErrorCheck( hipGetDeviceCount(&num_devices) );



	int hwthread;

	int num_threads = 0;

	int thread_id = 0;



	#pragma omp parallel default(shared)

	{

		num_threads = omp_get_num_threads();

	}



	if(num_devices == 0){

		#pragma omp parallel default(shared) private(hwthread, thread_id)

		{
@@ -81,7 +76,7 @@ int main(int argc, char *argv[]){ 
			// Get the PCIBusId for each GPU and use it to query for UUID

			hipErrorCheck( hipDeviceGetPCIBusId(busid, 64, i) );



			// Concatenate per-MPIrank GPU info into strings for printf

			// Concatenate per-MPIrank GPU info into strings for print

            if(i > 0) rt_gpu_id_list.append(",");

            rt_gpu_id_list.append(std::to_string(i));