Work on nuccor_kernels. pgi/c/crest passes.

Crest/ScaLAPACK/Source/Common_Scripts/build_executable.x

@@ -119,34 +119,32 @@ def is_cray():
 def use_syslapack(test):
     """Helper: do we use system blas/lapack or build our own."""
 
-    return re.search('_syslapack', test) != None
+    return re.search('_syslapack_', test+'_') != None
 
 #------------------------------------------------------------------------------
 
-COMPILER_GNU = 'GNU'
-COMPILER_PGI = 'PGI'
-COMPILER_CRAY = 'CRAY'
-COMPILER_INTEL = 'INTEL'
-COMPILER_LLVM = 'LLVM'
-COMPILER_XL = 'XL'
-
-#------------------------------------------------------------------------------
+COMPILER_TYPE_GNU = 'GNU'
+COMPILER_TYPE_PGI = 'PGI'
+COMPILER_TYPE_CRAY = 'CRAY'
+COMPILER_TYPE_INTEL = 'INTEL'
+COMPILER_TYPE_LLVM = 'LLVM'
+COMPILER_TYPE_XL = 'XL'
 
 def compiler_type(test):
     """Helper: which compiler are we using."""
 
-    if re.search('_gnu', test) != None:
-        return COMPILER_GNU
-    elif re.search('_pgi', test) != None:
-        return COMPILER_PGI
-    elif re.search('_cray', test) != None:
-        return COMPILER_CRAY
-    elif re.search('_intel', test) != None:
-        return COMPILER_INTEL
-    elif re.search('_llvm', test) != None:
-        return COMPILER_LLVM
-    elif re.search('_xl', test) != None:
-        return COMPILER_XL
+    if re.search('_gnu_', test+'_') != None:
+        return COMPILER_TYPE_GNU
+    elif re.search('_pgi_', test+'_') != None:
+        return COMPILER_TYPE_PGI
+    elif re.search('_cray_', test+'_') != None:
+        return COMPILER_TYPE_CRAY
+    elif re.search('_intel_', test+'_') != None:
+        return COMPILER_TYPE_INTEL
+    elif re.search('_llvm_', test+'_') != None:
+        return COMPILER_TYPE_LLVM
+    elif re.search('_xl_', test+'_') != None:
+        return COMPILER_TYPE_XL
     else:
         assert False, 'Compiler type not recognized. ' + test
 
@@ -173,13 +171,13 @@ def make_lapack(build_dir_path, test):
     make_inc = f.read()
     f.close()
 
-    make_command = 'set -e -o pipefail' # Set make command to exit fast if fail.
+    make_command = 'set -eu -o pipefail' # Set command to exit fast if fail.
 
     #--------------------
     if is_cray():
     #--------------------
         #--------------------
-        if compiler_type(test) == COMPILER_GNU:
+        if compiler_type(test) == COMPILER_TYPE_GNU:
         #--------------------
             substitutions = [
                 ['FORTRAN  = gfortran', 'FORTRAN  = ftn'],
@@ -202,7 +200,7 @@ def make_lapack(build_dir_path, test):
     else: # IBM.
     #--------------------
         #--------------------
-        if compiler_type(test) == COMPILER_GNU:
+        if compiler_type(test) == COMPILER_TYPE_GNU:
         #--------------------
             substitutions = [
                 ['FORTRAN  = gfortran', 'FORTRAN  = gfortran'],
@@ -299,13 +297,13 @@ def make_binary(build_dir_path, test):
     make_inc = f.read()
     f.close()
 
-    make_command = 'set -e -o pipefail' # Set make command to exit fast if fail.
+    make_command = 'set -eu -o pipefail' # Set command to exit fast if fail.
 
     #--------------------
     if is_cray():
     #--------------------
         #--------------------
-        if compiler_type(test) == COMPILER_GNU:
+        if compiler_type(test) == COMPILER_TYPE_GNU:
         #--------------------
             laflags = '-' + 'L' + os.path.join(build_dir_path, 'lapack')
             substitutions = [
@@ -333,21 +331,21 @@ def make_binary(build_dir_path, test):
             make_command += ';' + get_module_command('load PrgEnv-gnu')
 
         #--------------------
-        elif compiler_type(test) == COMPILER_PGI:
+        elif compiler_type(test) == COMPILER_TYPE_PGI:
         #--------------------
             substitutions = [
             ]
             assert False, 'Compiler not yet implemented. ' + compiler_type(test)
 
         #--------------------
-        elif compiler_type(test) == COMPILER_CRAY:
+        elif compiler_type(test) == COMPILER_TYPE_CRAY:
         #--------------------
             substitutions = [
             ]
             assert False, 'Compiler not yet implemented. ' + compiler_type(test)
 
         #--------------------
-        elif compiler_type(test) == COMPILER_INTEL:
+        elif compiler_type(test) == COMPILER_TYPE_INTEL:
         #--------------------
             substitutions = [
             ]
@@ -362,7 +360,7 @@ def make_binary(build_dir_path, test):
     else: # IBM.
     #--------------------
         #--------------------
-        if compiler_type(test) == COMPILER_GNU:
+        if compiler_type(test) == COMPILER_TYPE_GNU:
         #--------------------
             laflags = '-' + 'L' + lapack_dir_path
             #---Compatibility libs to use essl with gnu.
@@ -392,7 +390,7 @@ def make_binary(build_dir_path, test):
             ]
 
         #--------------------
-        elif compiler_type(test) == COMPILER_PGI:
+        elif compiler_type(test) == COMPILER_TYPE_PGI:
         #--------------------
             laflags = '-' + 'L' + lapack_dir_path
             #---Compatibility libs to use essl with gnu.
@@ -453,7 +451,7 @@ def make_binary(build_dir_path, test):
             make_command += ';' + get_module_command('load pgi')
 
         #--------------------
-        elif compiler_type(test) == COMPILER_LLVM:
+        elif compiler_type(test) == COMPILER_TYPE_LLVM:
         #--------------------
             # TODO
             substitutions = [
@@ -462,7 +460,7 @@ def make_binary(build_dir_path, test):
             make_command += ';' + get_module_command('load xlflang')
 
         #--------------------
-        elif compiler_type(test) == COMPILER_XL:
+        elif compiler_type(test) == COMPILER_TYPE_XL:
         #--------------------
             laflags = '-' + 'L' + lapack_dir_path
             esslflags = '-lessl'

Crest/ScaLAPACK/Source/make_fmod.sh

@@ -3,7 +3,7 @@
 
 # Exit immediately on error.
 
-set -e -o pipefail
+set -eu -o pipefail
 
 # Create files needed by xlf95.
 

Crest/ScaLAPACK/Source/make_lapack.sh

@@ -3,7 +3,7 @@
 
 # Exit immediately on error.
 
-set -e -o pipefail
+set -eu -o pipefail
 
 cd lapack
 make -j8 blaslib 2>&1 | tee out_make_blaslib.txt

Crest/ScaLAPACK/Source/make_scalapack.sh

@@ -3,7 +3,7 @@
 
 # Exit immediately on error.
 
-set -e -o pipefail
+set -eu -o pipefail
 
 # NOTE: build fails if -j requests multiple build processes.
 

Crest/nuccor_kernels/Source/Common_Scripts/build_executable.x

@@ -112,39 +112,53 @@ def prepare_to_make(path_to_workspace):
 #
 #------------------------------------------------------------------------------
 
-COMPILER_GNU = 'GNU'
-COMPILER_PGI = 'PGI'
-COMPILER_CRAY = 'CRAY'
-COMPILER_INTEL = 'INTEL'
-COMPILER_LLVM = 'LLVM'
-COMPILER_XL = 'XL'
-
-#------------------------------------------------------------------------------
+COMPILER_TYPE_GNU = 'GNU'
+COMPILER_TYPE_PGI = 'PGI'
+COMPILER_TYPE_CRAY = 'CRAY'
+COMPILER_TYPE_INTEL = 'INTEL'
+COMPILER_TYPE_LLVM = 'LLVM'
+COMPILER_TYPE_XL = 'XL'
 
 def compiler_type(test):
     """Helper: which compiler are we using."""
 
-    if re.search('_gnu', test) != None:
-        return COMPILER_GNU
-    elif re.search('_pgi', test) != None:
-        return COMPILER_PGI
-    elif re.search('_cray', test) != None:
-        return COMPILER_CRAY
-    elif re.search('_intel', test) != None:
-        return COMPILER_INTEL
-    elif re.search('_llvm', test) != None:
-        return COMPILER_LLVM
-    elif re.search('_xl', test) != None:
-        return COMPILER_XL
+    if re.search('_gnu_', test+'_') != None:
+        return COMPILER_TYPE_GNU
+    elif re.search('_pgi_', test+'_') != None:
+        return COMPILER_TYPE_PGI
+    elif re.search('_cray', test+'_') != None:
+        return COMPILER_TYPE_CRAY
+    elif re.search('_intel_', test+'_') != None:
+        return COMPILER_TYPE_INTEL
+    elif re.search('_llvm_', test+'_') != None:
+        return COMPILER_TYPE_LLVM
+    elif re.search('_xl_', test+'_') != None:
+        return COMPILER_TYPE_XL
     else:
         assert False, 'Compiler type not recognized. ' + test
 
 #------------------------------------------------------------------------------
 
+COMPILER_LANG_F = 'F'
+COMPILER_LANG_C = 'C'
+
+def compiler_lang(test):
+    """Helper: which language to use for the source code."""
+
+    if re.search('_f_', test+'_') != None:
+        return COMPILER_LANG_F
+    elif re.search('_c_', test+'_') != None:
+        return COMPILER_LANG_C
+    else:
+        assert False, 'Compiler language not recognized. ' + test
+
+#------------------------------------------------------------------------------
+
 def make_binary(build_dir_path, test):
     """Execute the make to build executable."""
 
-    make_command = './make.sh ' + compiler_type(test)
+    make_command = ('./make.sh ' + compiler_type(test) + ' ' +
+                    compiler_lang(test))
 
     exit_status = os.system(make_command)
 

Crest/nuccor_kernels/Source/Common_Scripts/check_executable.x

+#! /usr/bin/env python
+"""
+-------------------------------------------------------------------------------
+File:   check_executable.x
+Author: Arnold Tharrington (arnoldt@ornl.gov)
+Modified: Veronica G. Vergara Larrea, Wayne Joubert
+National Center for Computational Sciences, Scientific Computing Group.
+Oak Ridge National Laboratory
+Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
+-------------------------------------------------------------------------------
+"""
+
+import os
+import argparse
+import re
+
+#------------------------------------------------------------------------------
+
+IS_PASSING_YES = 1
+IS_PASSING_NO = 0
+
+#------------------------------------------------------------------------------
+
+def process_command_line_args():
+    """Get the command line arguments."""
+
+    command_description = (
+        'A program that checks the results located at <path_to_results>. '
+        'The check executable must write the status of the results to the '
+        'file Status/<test_id_string>/job_status.txt')
+
+    p_help = 'The absoulte path to the results of a test.'
+
+    i_help = 'The test id string.'
+
+    parser = argparse.ArgumentParser(description=command_description)
+    parser.add_argument('-p', help=p_help, required=True)
+    parser.add_argument('-i', help=i_help, required=True)
+
+    args = parser.parse_args()
+
+    return args
+
+#------------------------------------------------------------------------------
+
+def main():
+    """Main program for check operation.  Check the correctness of
+       the run results and report back.
+    """
+
+    # Get the command line arguments.
+
+    args = process_command_line_args()
+    path_to_results = args.p
+    #test_id = args.i
+
+    # Compare the results.
+
+    is_passing = check_results(path_to_results)
+
+    # Write the status of the results to job data file.
+
+    write_to_job_status_file(path_to_results, is_passing)
+
+#------------------------------------------------------------------------------
+
+def check_results(path_to_results):
+    """Perform the ciorrectness check of the results."""
+
+    # Make the file name paths to numbers squared.
+
+    file_path = os.path.join(path_to_results, "std.out.txt")
+
+    if not os.path.exists(file_path):
+        return IS_PASSING_NO
+
+    file_ = open(file_path, "r")
+    lines = file_.readlines()
+    file_.close()
+
+    num_failed = None
+
+    for line in lines:
+
+        match = re.findall(r'^FINAL.*? num_failures ([0-9]+)$', line)
+
+        if len(match) == 1:
+            num_failed = match[0]
+
+    if num_failed is None:
+        return IS_PASSING_NO
+
+    return IS_PASSING_YES if num_failed == '0' else IS_PASSING_NO
+
+#------------------------------------------------------------------------------
+
+def write_to_job_status_file(path_to_results, is_passing):
+    """Write the status of the results to job data file."""
+
+    # Get path.
+
+    dir_head1, dir_tail1 = os.path.split(path_to_results)
+    dir_head2, dir_tail2 = os.path.split(dir_head1)
+    file_path = os.path.join(dir_head2, 'Status', dir_tail1, 'job_status.txt')
+
+    file_ = open(file_path, 'w')
+
+    # Create the the string to write.
+
+    if is_passing == IS_PASSING_NO:
+        indicator = '1'
+    elif is_passing == IS_PASSING_YES:
+        indicator = '0'
+    elif is_passing >= 2:
+        indicator = '2'
+    string_ = '%s\n' % (indicator)
+
+    # Write the string.
+
+    file_.write(string_)
+    file_.close()
+
+#------------------------------------------------------------------------------
+
+if __name__ == '__main__':
+    main()
+
+#------------------------------------------------------------------------------

Crest/nuccor_kernels/Source/Common_Scripts/lsf.template.x

+#! /bin/bash -l
+#------------------------------------------------------------------------------
+#BSUB -q __batchqueue__
+#BSUB -J __jobname__
+#BSUB -o __resultsdir__/__jobname__.o%J
+#BSUB -e __resultsdir__/__jobname__.e%J
+#BSUB -n __nodes__
+#BSUB -W __walltime__
+#BSUB -b __starttime__
+
+#-----------------------------------------------------
+# Set up the environment for use of the harness.
+#-----------------------------------------------------
+source __rgtenvironmentalfile__
+module load __requiredmodules__
+module list
+
+#-----------------------------------------------------
+# Define some variables.
+#-----------------------------------------------------
+EXECUTABLE="__pathtoexecutable__"
+STARTINGDIRECTORY="__startingdirectory__"
+WORKDIR="__workdir__"
+RESULTSDIR="__resultsdir__"
+TEST_ID="__test_id__"
+
+#-----------------------------------------------------
+# Ensure that we are in the correct starting
+# directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+echo "Starting directory is $(pwd)"
+
+#-----------------------------------------------------
+# Make the working scratch space directory.
+#-----------------------------------------------------
+mkdir -p $WORKDIR
+
+#-----------------------------------------------------
+# Make the results directory if not already there.
+#-----------------------------------------------------
+#mkdir -p $RESULTSDIR
+
+#-----------------------------------------------------
+# Copy needed files to work dir.
+#-----------------------------------------------------
+#EXECUTABLE_DIR=$(dirname $EXECUTABLE)
+#for FILE in $EXECUTABLE $EXECUTABLE_DIR"/"*.dat ; do
+#  cp $FILE $WORKDIR
+#done
+
+cp ../Inputs/__inputfile__ $WORKDIR/sizes.txt
+
+#-----------------------------------------------------
+#  Change directory to the working directory.
+#-----------------------------------------------------
+cd $WORKDIR
+
+#-----------------------------------------------------
+# Run the executable.
+#-----------------------------------------------------
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode start
+
+__joblaunchcommand__
+
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode final
+
+sleep 30
+#-----------------------------------------------------
+# Ensure that we return to the starting directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+
+#-----------------------------------------------------
+# Copy the results back to the $RESULTSDIR.
+#-----------------------------------------------------
+cp -rf $WORKDIR"/"* $RESULTSDIR ### && rm -rf $WORKDIR
+
+#-----------------------------------------------------
+# Move the batch file to $RESULTSDIR.
+#-----------------------------------------------------
+if [ -e __batchfilename__ ] ; then
+  mv __batchfilename__ $RESULTSDIR
+fi
+
+#-----------------------------------------------------
+# Check the final results.
+#-----------------------------------------------------
+check_executable_driver.py -p $RESULTSDIR -i $TEST_ID
+
+#-----------------------------------------------------
+# The script now determines if we are to resubmit
+# itself.
+#-----------------------------------------------------
+case __resubmitme__ in
+    0) 
+       test_harness_driver.py -r;;
+
+    1) 
+       echo "No resubmit";;
+esac 
+
+#------------------------------------------------------------------------------

Crest/nuccor_kernels/Source/Common_Scripts/pbs.template.x

+#! /bin/bash -l
+#------------------------------------------------------------------------------
+#PBS -e __resultsdir__
+#PBS -o __resultsdir__
+#PBS -N __jobname__
+#PBS -l walltime=__walltime__
+#PBS -l nodes=__nodes__
+#PBS -q __batchqueue__
+#PBS -A __pbsaccountid__
+#PBS -a __starttime__
+
+#-----------------------------------------------------
+# Set up the environment for use of the harness.
+#-----------------------------------------------------
+source __rgtenvironmentalfile__
+module load __requiredmodules__
+module list
+
+#-----------------------------------------------------
+# Define some variables.
+#-----------------------------------------------------
+EXECUTABLE="__pathtoexecutable__"
+STARTINGDIRECTORY="__startingdirectory__"
+WORKDIR="__workdir__"
+RESULTSDIR="__resultsdir__"
+TEST_ID="__test_id__"
+
+#-----------------------------------------------------
+# Ensure that we are in the correct starting
+# directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+echo "Starting directory is $(pwd)"
+
+#-----------------------------------------------------
+# Make the working scratch space directory.
+#-----------------------------------------------------
+mkdir -p $WORKDIR
+
+#-----------------------------------------------------
+# Make the results directory if not already there.
+#-----------------------------------------------------
+#mkdir -p $RESULTSDIR
+
+#-----------------------------------------------------
+# Copy needed files to work dir.
+#-----------------------------------------------------
+#EXECUTABLE_DIR=$(dirname $EXECUTABLE)
+#for FILE in $EXECUTABLE $EXECUTABLE_DIR"/"*.dat ; do
+#  cp $FILE $WORKDIR
+#done
+
+cp ../Inputs/__inputfile__ $WORKDIR/sizes.txt
+
+#-----------------------------------------------------
+#  Change directory to the working directory.
+#-----------------------------------------------------
+cd $WORKDIR
+
+#-----------------------------------------------------
+# Run the executable.
+#-----------------------------------------------------
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode start
+
+__joblaunchcommand__
+
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode final
+
+sleep 30
+#-----------------------------------------------------
+# Ensure that we return to the starting directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+
+#-----------------------------------------------------
+# Copy the results back to the $RESULTSDIR.
+#-----------------------------------------------------
+cp -rf $WORKDIR"/"* $RESULTSDIR ### && rm -rf $WORKDIR
+
+#-----------------------------------------------------
+# Move the batch file to $RESULTSDIR.
+#-----------------------------------------------------
+if [ -e __batchfilename__ ] ; then
+  mv __batchfilename__ $RESULTSDIR
+fi
+
+#-----------------------------------------------------
+# Check the final results.
+#-----------------------------------------------------
+check_executable_driver.py -p $RESULTSDIR -i $TEST_ID
+
+#-----------------------------------------------------
+# The script now determines if we are to resubmit
+# itself.
+#-----------------------------------------------------
+case __resubmitme__ in
+    0) 
+       test_harness_driver.py -r;;
+
+    1) 
+       echo "No resubmit";;
+esac