Working code set up for ScaLAPACK testing.

Chester/ScaLAPACK

+../Crest/ScaLAPACK
\ No newline at end of file

Crest/ScaLAPACK/Source/Common_Scripts/build_executable.x

+#!/usr/bin/env python
+"""
+-------------------------------------------------------------------------------
+File:   build_executable.x
+Author: Wayne Joubert (joubert@ornl.gov)
+National Center for Computational Sciences, Scientific Computing Group.
+Oak Ridge National Laboratory
+Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
+-------------------------------------------------------------------------------
+"""
+
+import os
+import argparse
+import shutil
+import re
+
+#------------------------------------------------------------------------------
+
+def process_command_line_args():
+    """Get the command line arguments."""
+
+    command_description = (
+        'A driver program that builds the binary for the test.')
+
+    p_help = (
+        'The absolute path to the workspace. This path must have the '
+        'appropiate permissions to permit the user of the test to r, w, and x.')
+
+    i_help = (
+        'The test id string. The build program uses this string to make a '
+        'unique directory within path_to_workspace. We don\'t want concurrent '
+        'builds to clobber each other.  The submit program uses this string '
+        'to write the job schedule id to Status/<test_id_string>/job_id.txt')
+
+    parser = argparse.ArgumentParser(description=command_description)
+    parser.add_argument('-p', help=p_help, required=True)
+    parser.add_argument('-i', help=i_help, required=True)
+
+    args = parser.parse_args()
+
+    return args
+
+#------------------------------------------------------------------------------
+
+def main():
+    """Main program for building the executable."""
+
+    # Get the command line arguments.
+
+    args = process_command_line_args()
+    path_to_workspace = args.p
+    #test_id = args.i
+
+    # Create the temporary workspace.
+    # Save the tempoary workspace for the submit executable.
+
+    create_tmp_workspace(path_to_workspace)
+
+    # Make the binary.
+
+    prepare_to_make(path_to_workspace)
+
+    exit_status = make_lapack()
+    if exit_status != 0:
+        return 1
+
+    exit_status = make_binary()
+    if exit_status != 0:
+        return 1
+
+    return 0
+
+#------------------------------------------------------------------------------
+
+def prepare_to_make(path_to_workspace):
+    """Perform preparations, e.g., copying source tree."""
+
+    # Get the current working directory.
+
+    cwd = os.getcwd()
+
+    # Get the 2 tail paths in the cwd.
+
+    dir_head1, dir_tail1 = os.path.split(cwd)
+    dir_head2, dir_tail2 = os.path.split(dir_head1)
+
+    # Get the path to the Source directory for the application.
+
+    path_to_source = os.path.join(dir_head2, 'Source')
+
+    # Now make the path to the build directory.
+
+    build_dir_path = os.path.join(path_to_workspace, 'build_directory')
+
+    # Copy Source to build directory.
+
+    print('Copying source tree ...')
+    shutil.copytree(path_to_source, build_dir_path, symlinks=True)
+
+    # Change to build directory.
+
+    os.chdir(build_dir_path)
+
+#------------------------------------------------------------------------------
+
+def is_cray():
+    """Helper: are we on a Cray system."""
+
+    return 'CRAYPE_VERSION' in os.environ
+
+#------------------------------------------------------------------------------
+
+COMPILER_GNU = 'GNU'
+COMPILER_PGI = 'PGI'
+COMPILER_CRAY = 'CRAY'
+COMPILER_INTEL = 'INTEL'
+COMPILER_LLVM = 'LLVM'
+COMPILER_XL = 'XL'
+
+#------------------------------------------------------------------------------
+
+def compiler_type(test):
+    """Helper: which compiler are we using."""
+
+    if re.search('_gnu', test) != None:
+        return COMPILER_GNU
+    elif re.search('_pgi', test) != None:
+        return COMPILER_PGI
+    elif re.search('_cray', test) != None:
+        return COMPILER_CRAY
+    elif re.search('_intel', test) != None:
+        return COMPILER_INTEL
+    elif re.search('_llvm', test) != None:
+        return COMPILER_LLVM
+    elif re.search('_xl', test) != None:
+        return COMPILER_XL
+    else:
+        assert False, 'Compiler type not recognized. ' + test
+
+#------------------------------------------------------------------------------
+
+def get_module_command(module_command_args):
+    """Helper: et the shell code to do a module request."""
+
+    modules_home = os.environ['MODULESHOME']
+    module_cmd = modules_home + '/bin/modulecmd'
+    return 'eval `' + module_cmd + ' sh ' + module_command_args + '`'
+
+#------------------------------------------------------------------------------
+
+def make_lapack():
+    """Execute the make to build blas and lapack with test drivers."""
+
+    # Get dir names etc.
+
+    build_dir_path = os.getcwd()
+    dir_head1, _ = os.path.split(build_dir_path)
+    dir_head2, _ = os.path.split(dir_head1)
+    _, test = os.path.split(dir_head2)
+
+    # Modify makefile.
+
+    f = open(os.path.join(build_dir_path, 'lapack', 'make.inc.example'), 'r')
+    make_inc = f.read()
+    f.close()
+
+    make_command = 'true'
+
+    if is_cray():
+        if compiler_type(test) == COMPILER_GNU:
+            replacements = [
+                [r'FORTRAN  = gfortran', r'FORTRAN  = ftn'],
+                [r'CC = gcc', r'CC = cc'],
+                [r'OPTS     = -O2', r'OPTS     = -O2'],
+                [r'NOOPT    = -O0', r'NOOPT    = -O0'],
+                [r'LOADER   = gfortran', r'LOADER   = ftn'],
+                [r'LOADOPTS =', r'LOADOPTS ='],
+                [r'TIMER    = INT_ETIME', r'TIMER    = INT_ETIME'],
+            ]
+            make_command += ';' + get_module_command('unload PrgEnv-pgi')
+            make_command += ';' + get_module_command('load PrgEnv-gnu')
+
+    else: # IBM.
+        if compiler_type(test) == COMPILER_GNU:
+            replacements = [
+                [r'FORTRAN  = gfortran', r'FORTRAN  = gfortran'],
+                [r'CC = gcc', r'CC = gcc'],
+                [r'OPTS     = -O2', r'OPTS     = -O2'],
+                [r'NOOPT    = -O0', r'NOOPT    = -O0'],
+                [r'LOADER   = gfortran', r'LOADER   = gfortran'],
+                [r'LOADOPTS =', r'LOADOPTS ='],
+                [r'TIMER    = INT_ETIME', r'TIMER    = INT_ETIME'],
+            ]
+        elif compiler_type(test) == 'PGI':
+            replacements = [
+            ]
+        elif compiler_type(test) == 'LLVM':
+            replacements = [
+            ]
+        elif compiler_type(test) == 'XL':
+            replacements = [
+            ]
+        else:
+            assert False, 'Compiler type not recognized. ' + compiler_type(test)
+
+    for replacement in replacements:
+        make_inc = re.sub(replacement[0], replacement[1], make_inc)
+
+    f = open(os.path.join(build_dir_path, 'lapack', 'make.inc'), 'w')
+    f.write(make_inc)
+    f.close()
+
+    make_command += ';' + get_module_command('list')
+    make_command += ';' + './make_lapack.sh'
+
+    exit_status = os.system(make_command)
+
+    if os.path.exists(os.path.join(build_dir_path, 'lapack', 'librefblas.a')):
+        shutil.copyfile(os.path.join(build_dir_path, 'lapack', 'librefblas.a'),
+                        os.path.join(build_dir_path, 'lapack', 'libblas.a'))
+
+    return exit_status
+
+#------------------------------------------------------------------------------
+
+def make_binary():
+    """Execute the make to build library and test drivers."""
+
+    # Get dir names etc.
+
+    build_dir_path = os.getcwd()
+    dir_head1, _ = os.path.split(build_dir_path)
+    dir_head2, _ = os.path.split(dir_head1)
+    _, test = os.path.split(dir_head2)
+    lapack_dir_path = os.path.join(build_dir_path, 'lapack')
+
+    # Modify makefile.
+
+    f = open(os.path.join(build_dir_path, 'scalapack',
+                          'SLmake.inc.example'), 'r')
+    make_inc = f.read()
+    f.close()
+
+    make_command = 'true'
+
+    if is_cray():
+        if compiler_type(test) == COMPILER_GNU:
+            replacements = [
+                [r'CDEFS         = -DAdd_', r'CDEFS         = -DAdd_'],
+                [r'FC            = mpif90', r'FC            = ftn'],
+                [r'CC            = mpicc', r'CC            = cc'],
+                [r'FCFLAGS       = -O3', r'FCFLAGS       = -O3 -fPIC'],
+                [r'CCFLAGS       = -O3', r'CCFLAGS       = -O3 -fPIC'],
+                [r'FCLOADFLAGS   = ..FCFLAGS.',
+                 r'FCLOADFLAGS   = $(FCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'],
+                [r'CCLOADFLAGS   = ..CCFLAGS.',
+                 r'CCLOADFLAGS   = $(CCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'],
+                [r'BLASLIB       = -lblas', r'BLASLIB       = -lsci_gnu'],
+                [r'LAPACKLIB     = -llapack', r'LAPACKLIB     = -lsci_gnu'],
+            ]
+            make_command += ';' + get_module_command('unload PrgEnv-pgi')
+            make_command += ';' + get_module_command('load PrgEnv-gnu')
+            make_command += ';' + get_module_command('load cray-libsci')
+
+        elif compiler_type(test) == COMPILER_PGI:
+            replacements = [
+            ]
+            assert False, 'Compiler not yet implemented. ' + compiler_type(test)
+
+        elif compiler_type(test) == COMPILER_CRAY:
+            replacements = [
+            ]
+            assert False, 'Compiler not yet implemented. ' + compiler_type(test)
+
+        elif compiler_type(test) == COMPILER_INTEL:
+            replacements = [
+            ]
+            assert False, 'Compiler not yet implemented. ' + compiler_type(test)
+
+        else:
+            assert False, 'Compiler type not recognized. ' + compiler_type(test)
+
+    else: # IBM.
+        if compiler_type(test) == COMPILER_GNU:
+            replacements = [
+                [r'CDEFS         = -DAdd_', r'CDEFS         = -DAdd_'],
+                [r'FC            = mpif90', r'FC            = mpfort'],
+                [r'CC            = mpicc', r'CC            = mpcc'],
+                [r'FCFLAGS       = -O3', r'FCFLAGS       = -O3 -compiler gnu'],
+                [r'CCFLAGS       = -O3', r'CCFLAGS       = -O3 -compiler gnu'],
+                [r'NOOPT         = -O0', r'NOOPT         = -O0 -compiler gnu'],
+                [r'FCLOADFLAGS   = ..FCFLAGS.',
+                 r'FCLOADFLAGS   = $(FCFLAGS) -L' +
+                 os.path.join(build_dir_path, 'lapack')],
+                [r'CCLOADFLAGS   = ..CCFLAGS.',
+                 r'CCLOADFLAGS   = $(CCFLAGS) -L' +
+                 os.path.join(build_dir_path, 'lapack')],
+            ]
+
+        elif compiler_type(test) == COMPILER_PGI:
+            replacements = [
+            ]
+
+        elif compiler_type(test) == COMPILER_LLVM:
+            replacements = [
+            ]
+
+        elif compiler_type(test) == COMPILER_XL:
+            replacements = [
+            ]
+
+        else:
+            assert False, 'Compiler type not recognized. ' + compiler_type(test)
+
+    for replacement in replacements:
+        make_inc = re.sub(replacement[0], replacement[1], make_inc)
+
+    f = open(os.path.join(build_dir_path, 'scalapack', 'SLmake.inc'), 'w')
+    f.write(make_inc)
+    f.close()
+
+    make_command += ';' + get_module_command('list')
+    make_command += ';' + './make_scalapack.sh'
+
+    exit_status = os.system(make_command)
+
+    return exit_status
+
+#------------------------------------------------------------------------------
+
+def create_tmp_workspace(path1):
+    """Create the workspace dir."""
+    os.makedirs(path1)
+
+#------------------------------------------------------------------------------
+
+if __name__ == '__main__':
+    main()
+
+#------------------------------------------------------------------------------

Crest/ScaLAPACK/Source/Common_Scripts/check_executable.x

+#! /usr/bin/env python
+"""
+-------------------------------------------------------------------------------
+File:   check_executable.x
+Author: Arnold Tharrington (arnoldt@ornl.gov)
+Modified: Veronica G. Vergara Larrea, Wayne Joubert
+National Center for Computational Sciences, Scientific Computing Group.
+Oak Ridge National Laboratory
+Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
+-------------------------------------------------------------------------------
+"""
+
+import os
+import argparse
+import re
+
+#------------------------------------------------------------------------------
+
+IS_PASSING_YES = 1
+IS_PASSING_NO = 0
+
+#------------------------------------------------------------------------------
+
+def process_command_line_args():
+    """Get the command line arguments."""
+
+    command_description = (
+        'A program that checks the results located at <path_to_results>. '
+        'The check executable must write the status of the results to the '
+        'file Status/<test_id_string>/job_status.txt')
+
+    p_help = 'The absoulte path to the results of a test.'
+
+    i_help = 'The test id string.'
+
+    parser = argparse.ArgumentParser(description=command_description)
+    parser.add_argument('-p', help=p_help, required=True)
+    parser.add_argument('-i', help=i_help, required=True)
+
+    args = parser.parse_args()
+
+    return args
+
+#------------------------------------------------------------------------------
+
+def main():
+    """Main program for check operation.  Check the correctness of
+       the run results and report back.
+    """
+
+    # Get the command line arguments.
+
+    args = process_command_line_args()
+    path_to_results = args.p
+    #test_id = args.i
+
+    # Compare the results.
+
+    is_passing = check_results(path_to_results)
+
+    # Write the status of the results to job data file.
+
+    write_to_job_status_file(path_to_results, is_passing)
+
+#------------------------------------------------------------------------------
+
+def check_results(path_to_results):
+    """Perform the ciorrectness check of the results."""
+
+    # Make the file name paths to numbers squared.
+
+    file_path = os.path.join(path_to_results, "std.out.txt")
+
+    if not os.path.exists(file_path):
+        return IS_PASSING_NO
+
+    file_ = open(file_path, "r")
+    lines = file_.readlines()
+    file_.close()
+
+    num_total = None
+    num_passed = None
+
+    for line in lines:
+
+        match = re.findall('^Finished *(\d+) tests, '
+                           'with the following results.*', line)
+        if len(match) == 1:
+            num_total = match
+
+        match = re.findall('^ *(\d+) tests completed and '
+                           'passed residual checks.*', line)
+        if len(match) == 1:
+            num_passed = match
+
+    if num_total is None:
+        return IS_PASSING_NO
+
+    if num_passed is None:
+        return IS_PASSING_NO
+
+    return IS_PASSING_YES if num_passed == num_total else IS_PASSING_NO
+
+#------------------------------------------------------------------------------
+
+def write_to_job_status_file(path_to_results, is_passing):
+    """Write the status of the results to job data file."""
+
+    # Get path.
+
+    dir_head1, dir_tail1 = os.path.split(path_to_results)
+    dir_head2, dir_tail2 = os.path.split(dir_head1)
+    file_path = os.path.join(dir_head2, 'Status', dir_tail1, 'job_status.txt')
+
+    file_ = open(file_path, 'w')
+
+    # Create the the string to write.
+
+    if is_passing == IS_PASSING_NO:
+        indicator = '1'
+    elif is_passing == IS_PASSING_YES:
+        indicator = '0'
+    elif is_passing >= 2:
+        indicator = '2'
+    string_ = '%s\n' % (indicator)
+
+    # Write the string.
+
+    file_.write(string_)
+    file_.close()
+
+#------------------------------------------------------------------------------
+
+if __name__ == '__main__':
+    main()
+
+#------------------------------------------------------------------------------

Crest/ScaLAPACK/Source/Common_Scripts/lsf.template.x

+#! /bin/bash -l
+#------------------------------------------------------------------------------
+#BSUB -q __batchqueue__
+#BSUB -J __jobname__
+#BSUB -o __resultsdir__/__jobname__.o%J
+#BSUB -e __resultsdir__/__jobname__.e%J
+#BSUB -n __nodes__
+#BSUB -W __walltime__
+#BSUB -b __starttime__
+
+#-----------------------------------------------------
+# Set up the environment for use of the harness.
+#-----------------------------------------------------
+source __rgtenvironmentalfile__
+module load __nccstestharnessmodule__
+module list
+
+#-----------------------------------------------------
+# Define some variables.
+#-----------------------------------------------------
+EXECUTABLE="__pathtoexecutable__"
+STARTINGDIRECTORY="__startingdirectory__"
+WORKDIR="__workdir__"
+RESULTSDIR="__resultsdir__"
+TEST_ID="__test_id__"
+
+#-----------------------------------------------------
+# Ensure that we are in the correct starting
+# directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+echo "Starting directory is $(pwd)"
+
+#-----------------------------------------------------
+# Make the working scratch space directory.
+#-----------------------------------------------------
+mkdir -p $WORKDIR
+
+#-----------------------------------------------------
+# Make the results directory if not already there.
+#-----------------------------------------------------
+#mkdir -p $RESULTSDIR
+
+#-----------------------------------------------------
+# Copy needed files to work dir.
+#-----------------------------------------------------
+EXECUTABLE_DIR=$(dirname $EXECUTABLE)
+for FILE in $EXECUTABLE $EXECUTABLE_DIR"/"*.dat ; do
+  cp $FILE $WORKDIR
+done
+
+#-----------------------------------------------------
+#  Change directory to the working directory.
+#-----------------------------------------------------
+cd $WORKDIR
+
+#-----------------------------------------------------
+# Run the executable.
+#-----------------------------------------------------
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode start
+
+__joblaunchcommand__
+
+log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
+                             --uniqueid $TEST_ID --mode final
+
+sleep 30
+#-----------------------------------------------------
+# Ensure that we return to the starting directory.
+#-----------------------------------------------------
+cd $STARTINGDIRECTORY
+
+#-----------------------------------------------------
+# Copy the results back to the $RESULTSDIR.
+#-----------------------------------------------------
+cp -rf $WORKDIR"/"* $RESULTSDIR ### && rm -rf $WORKDIR
+
+#-----------------------------------------------------
+# Move the batch file to $RESULTSDIR.
+#-----------------------------------------------------
+if [ -e __batchfilename__ ] ; then
+  mv __batchfilename__ $RESULTSDIR
+fi
+
+#-----------------------------------------------------
+# Check the final results.
+#-----------------------------------------------------
+check_executable_driver.py -p $RESULTSDIR -i $TEST_ID
+
+#-----------------------------------------------------
+# The script now determines if we are to resubmit
+# itself.
+#-----------------------------------------------------
+case __resubmitme__ in
+    0) 
+       test_harness_driver.py -r;;
+
+    1) 
+       echo "No resubmit";;
+esac 
+
+#------------------------------------------------------------------------------

Crest/ScaLAPACK/Source/Common_Scripts/pbs.template.x

+#! /bin/bash -l
+#------------------------------------------------------------------------------
+#PBS -e __resultsdir__
+#PBS -o __resultsdir__
+#PBS -N __jobname__
+#PBS -l walltime=__walltime__
+#PBS -l nodes=__nodes__
+#PBS -q __batchqueue__
+#PBS -A __pbsaccountid__