nuccor_kernels f/c pgi crest now works (using workarounds).

Crest/nuccor_kernels/Source/Common_Scripts/check_executable.x

@@ -82,7 +82,7 @@ def check_results(path_to_results):
 
     for line in lines:
 
-        match = re.findall(r'^FINAL.*? num_failures ([0-9]+)$', line)
+        match = re.findall(r'^ *FINAL.*? num_failures +([0-9]+) *$', line)
 
         if len(match) == 1:
             num_failed = match[0]

Crest/nuccor_kernels/Source/Common_Scripts/submit_executable.x

@@ -181,7 +181,8 @@ def make_batch_script(batch_recursive_mode, path_to_workspace,
         joblaunchcommand = '$EXECUTABLE 1> std.out.txt 2> std.err.txt'
 
     input_file = 'sizes_1128.txt'
-    input_file = 'sizes_1.txt'
+    #input_file = 'sizes_10.txt'
+    #input_file = 'sizes_1.txt'
 
     rg_array = [('__jobname__', jobname),
                 ('__walltime__', walltime),

Crest/nuccor_kernels/Source/make.sh

@@ -120,6 +120,9 @@ function make_nuccor_kernels
     #--------------------
     if [ $COMPILER_TYPE = PGI ] ; then
     #--------------------
+
+      # TODO: fix this
+
       echo "Compiling with PGI compiler, C version ..."
       EXECUTABLE=nuccor_dgemm_titan_c_pgi
       module load cudatoolkit
@@ -165,6 +168,8 @@ function make_nuccor_kernels
       module swap PrgEnv-cray PrgEnv-pgi
       module unload craype-accel-nvidia35
       rm -f *.o
+
+
     #--------------------
     fi
   #--------------------
@@ -202,10 +207,22 @@ function make_nuccor_kernels
     #--------------------
 
       echo "Compiling with $COMPILER_TYPE compiler, $COMPILER_LANG version ..."
+      module load pgi
+      module load cuda
 
+      local CFLAGS="-fast -acc -ta=nvidia:rdc -Minfo=accel"
+      local CUDAFLAGS="-L$CUDA_DIR/lib64/ -lcublas_static"
+      local CUDAFLAGS="$CUDAFLAGS -lcudart -lculibos -ldl -lpthread"
+      pgcc -c get_wall_time.c
+      pgf90 -c nuccor_dgemm.F90 $CFLAGS $CUDAFLAGS
 
+      pgf90 -o $EXECUTABLE nuccor_dgemm.o get_wall_time.o \
+        $CFLAGS $CUDAFLAGS $LAPACK_FLAGS -lcudaforblas
 
+      module unload cuda
+      module unload pgi
 
+      rm -f *.o
 
     #--------------------
     else

Crest/nuccor_kernels/Source/nuccor_dgemm.F90

@@ -250,8 +250,10 @@
         !$acc end host_data
         !$acc exit data delete(a3)
         !$acc exit data delete(a2)
-        !$acc exit data
-        ! !$acc exit data copyout(b)
+        ! vvv TODO: FIX
+        ! !$acc exit data
+        !$acc exit data copyout(b)
+        ! ^^^ TODO: FIX
         !$acc wait(0)
       endif
 
@@ -280,8 +282,10 @@
         !$acc end host_data
         !$acc exit data delete(a3)
         !$acc exit data delete(a2)
-        !$acc exit data
-        ! !$acc exit data copyout(b)
+        ! vvv TODO: FIX
+        ! !$acc exit data
+        !$acc exit data copyout(b)
+        ! ^^^ TODO: FIX
         !$acc wait(0)
       endif
 

Crest/nuccor_kernels/Source/nuccor_dgemm.c

@@ -236,9 +236,10 @@ void compute_product_gpu(Matrix* c, Matrix* a1, Matrix* a2, Matrix* a3,
 
     #pragma acc exit data delete(a3_data[:m*n])
     #pragma acc exit data delete(a2_data[:m*m])
-/*
-    #pragma acc exit data copyout(bp_data[:m*n])
-*/
+    // vvv TODO: FIX
+    //#pragma acc exit data
+     #pragma acc exit data copyout(bp_data[:m*n])
+    // ^^^ TODO: FIX
     #pragma acc wait(0)
   }
 

Crest/nuccor_kernels/test_1node_f_pgi/Inputs

+../Source/Inputs
\ No newline at end of file

Crest/nuccor_kernels/test_1node_f_pgi/Scripts/build_executable.x

+../../Source/Common_Scripts/build_executable.x
\ No newline at end of file

Crest/nuccor_kernels/test_1node_f_pgi/Scripts/check_executable.x

+../../Source/Common_Scripts/check_executable.x
\ No newline at end of file

Crest/nuccor_kernels/test_1node_f_pgi/Scripts/lsf.template.x

+../../Source/Common_Scripts/lsf.template.x
\ No newline at end of file

Crest/nuccor_kernels/test_1node_f_pgi/Scripts/pbs.template.x

+../../Source/Common_Scripts/pbs.template.x
\ No newline at end of file

Crest/nuccor_kernels/test_1node_f_pgi/Scripts/submit_executable.x

+../../Source/Common_Scripts/submit_executable.x
\ No newline at end of file