Commit 03d9f52c authored by Joubert, Wayne's avatar Joubert, Wayne
Browse files

Working code set up for ScaLAPACK testing.

parent 77ba4736
../Crest/ScaLAPACK
\ No newline at end of file
#!/usr/bin/env python
"""
-------------------------------------------------------------------------------
File: build_executable.x
Author: Wayne Joubert (joubert@ornl.gov)
National Center for Computational Sciences, Scientific Computing Group.
Oak Ridge National Laboratory
Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
-------------------------------------------------------------------------------
"""
import os
import argparse
import shutil
import re
#------------------------------------------------------------------------------
def process_command_line_args():
"""Get the command line arguments."""
command_description = (
'A driver program that builds the binary for the test.')
p_help = (
'The absolute path to the workspace. This path must have the '
'appropiate permissions to permit the user of the test to r, w, and x.')
i_help = (
'The test id string. The build program uses this string to make a '
'unique directory within path_to_workspace. We don\'t want concurrent '
'builds to clobber each other. The submit program uses this string '
'to write the job schedule id to Status/<test_id_string>/job_id.txt')
parser = argparse.ArgumentParser(description=command_description)
parser.add_argument('-p', help=p_help, required=True)
parser.add_argument('-i', help=i_help, required=True)
args = parser.parse_args()
return args
#------------------------------------------------------------------------------
def main():
"""Main program for building the executable."""
# Get the command line arguments.
args = process_command_line_args()
path_to_workspace = args.p
#test_id = args.i
# Create the temporary workspace.
# Save the tempoary workspace for the submit executable.
create_tmp_workspace(path_to_workspace)
# Make the binary.
prepare_to_make(path_to_workspace)
exit_status = make_lapack()
if exit_status != 0:
return 1
exit_status = make_binary()
if exit_status != 0:
return 1
return 0
#------------------------------------------------------------------------------
def prepare_to_make(path_to_workspace):
"""Perform preparations, e.g., copying source tree."""
# Get the current working directory.
cwd = os.getcwd()
# Get the 2 tail paths in the cwd.
dir_head1, dir_tail1 = os.path.split(cwd)
dir_head2, dir_tail2 = os.path.split(dir_head1)
# Get the path to the Source directory for the application.
path_to_source = os.path.join(dir_head2, 'Source')
# Now make the path to the build directory.
build_dir_path = os.path.join(path_to_workspace, 'build_directory')
# Copy Source to build directory.
print('Copying source tree ...')
shutil.copytree(path_to_source, build_dir_path, symlinks=True)
# Change to build directory.
os.chdir(build_dir_path)
#------------------------------------------------------------------------------
def is_cray():
"""Helper: are we on a Cray system."""
return 'CRAYPE_VERSION' in os.environ
#------------------------------------------------------------------------------
COMPILER_GNU = 'GNU'
COMPILER_PGI = 'PGI'
COMPILER_CRAY = 'CRAY'
COMPILER_INTEL = 'INTEL'
COMPILER_LLVM = 'LLVM'
COMPILER_XL = 'XL'
#------------------------------------------------------------------------------
def compiler_type(test):
"""Helper: which compiler are we using."""
if re.search('_gnu', test) != None:
return COMPILER_GNU
elif re.search('_pgi', test) != None:
return COMPILER_PGI
elif re.search('_cray', test) != None:
return COMPILER_CRAY
elif re.search('_intel', test) != None:
return COMPILER_INTEL
elif re.search('_llvm', test) != None:
return COMPILER_LLVM
elif re.search('_xl', test) != None:
return COMPILER_XL
else:
assert False, 'Compiler type not recognized. ' + test
#------------------------------------------------------------------------------
def get_module_command(module_command_args):
"""Helper: et the shell code to do a module request."""
modules_home = os.environ['MODULESHOME']
module_cmd = modules_home + '/bin/modulecmd'
return 'eval `' + module_cmd + ' sh ' + module_command_args + '`'
#------------------------------------------------------------------------------
def make_lapack():
"""Execute the make to build blas and lapack with test drivers."""
# Get dir names etc.
build_dir_path = os.getcwd()
dir_head1, _ = os.path.split(build_dir_path)
dir_head2, _ = os.path.split(dir_head1)
_, test = os.path.split(dir_head2)
# Modify makefile.
f = open(os.path.join(build_dir_path, 'lapack', 'make.inc.example'), 'r')
make_inc = f.read()
f.close()
make_command = 'true'
if is_cray():
if compiler_type(test) == COMPILER_GNU:
replacements = [
[r'FORTRAN = gfortran', r'FORTRAN = ftn'],
[r'CC = gcc', r'CC = cc'],
[r'OPTS = -O2', r'OPTS = -O2'],
[r'NOOPT = -O0', r'NOOPT = -O0'],
[r'LOADER = gfortran', r'LOADER = ftn'],
[r'LOADOPTS =', r'LOADOPTS ='],
[r'TIMER = INT_ETIME', r'TIMER = INT_ETIME'],
]
make_command += ';' + get_module_command('unload PrgEnv-pgi')
make_command += ';' + get_module_command('load PrgEnv-gnu')
else: # IBM.
if compiler_type(test) == COMPILER_GNU:
replacements = [
[r'FORTRAN = gfortran', r'FORTRAN = gfortran'],
[r'CC = gcc', r'CC = gcc'],
[r'OPTS = -O2', r'OPTS = -O2'],
[r'NOOPT = -O0', r'NOOPT = -O0'],
[r'LOADER = gfortran', r'LOADER = gfortran'],
[r'LOADOPTS =', r'LOADOPTS ='],
[r'TIMER = INT_ETIME', r'TIMER = INT_ETIME'],
]
elif compiler_type(test) == 'PGI':
replacements = [
]
elif compiler_type(test) == 'LLVM':
replacements = [
]
elif compiler_type(test) == 'XL':
replacements = [
]
else:
assert False, 'Compiler type not recognized. ' + compiler_type(test)
for replacement in replacements:
make_inc = re.sub(replacement[0], replacement[1], make_inc)
f = open(os.path.join(build_dir_path, 'lapack', 'make.inc'), 'w')
f.write(make_inc)
f.close()
make_command += ';' + get_module_command('list')
make_command += ';' + './make_lapack.sh'
exit_status = os.system(make_command)
if os.path.exists(os.path.join(build_dir_path, 'lapack', 'librefblas.a')):
shutil.copyfile(os.path.join(build_dir_path, 'lapack', 'librefblas.a'),
os.path.join(build_dir_path, 'lapack', 'libblas.a'))
return exit_status
#------------------------------------------------------------------------------
def make_binary():
"""Execute the make to build library and test drivers."""
# Get dir names etc.
build_dir_path = os.getcwd()
dir_head1, _ = os.path.split(build_dir_path)
dir_head2, _ = os.path.split(dir_head1)
_, test = os.path.split(dir_head2)
lapack_dir_path = os.path.join(build_dir_path, 'lapack')
# Modify makefile.
f = open(os.path.join(build_dir_path, 'scalapack',
'SLmake.inc.example'), 'r')
make_inc = f.read()
f.close()
make_command = 'true'
if is_cray():
if compiler_type(test) == COMPILER_GNU:
replacements = [
[r'CDEFS = -DAdd_', r'CDEFS = -DAdd_'],
[r'FC = mpif90', r'FC = ftn'],
[r'CC = mpicc', r'CC = cc'],
[r'FCFLAGS = -O3', r'FCFLAGS = -O3 -fPIC'],
[r'CCFLAGS = -O3', r'CCFLAGS = -O3 -fPIC'],
[r'FCLOADFLAGS = ..FCFLAGS.',
r'FCLOADFLAGS = $(FCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'],
[r'CCLOADFLAGS = ..CCFLAGS.',
r'CCLOADFLAGS = $(CCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'],
[r'BLASLIB = -lblas', r'BLASLIB = -lsci_gnu'],
[r'LAPACKLIB = -llapack', r'LAPACKLIB = -lsci_gnu'],
]
make_command += ';' + get_module_command('unload PrgEnv-pgi')
make_command += ';' + get_module_command('load PrgEnv-gnu')
make_command += ';' + get_module_command('load cray-libsci')
elif compiler_type(test) == COMPILER_PGI:
replacements = [
]
assert False, 'Compiler not yet implemented. ' + compiler_type(test)
elif compiler_type(test) == COMPILER_CRAY:
replacements = [
]
assert False, 'Compiler not yet implemented. ' + compiler_type(test)
elif compiler_type(test) == COMPILER_INTEL:
replacements = [
]
assert False, 'Compiler not yet implemented. ' + compiler_type(test)
else:
assert False, 'Compiler type not recognized. ' + compiler_type(test)
else: # IBM.
if compiler_type(test) == COMPILER_GNU:
replacements = [
[r'CDEFS = -DAdd_', r'CDEFS = -DAdd_'],
[r'FC = mpif90', r'FC = mpfort'],
[r'CC = mpicc', r'CC = mpcc'],
[r'FCFLAGS = -O3', r'FCFLAGS = -O3 -compiler gnu'],
[r'CCFLAGS = -O3', r'CCFLAGS = -O3 -compiler gnu'],
[r'NOOPT = -O0', r'NOOPT = -O0 -compiler gnu'],
[r'FCLOADFLAGS = ..FCFLAGS.',
r'FCLOADFLAGS = $(FCFLAGS) -L' +
os.path.join(build_dir_path, 'lapack')],
[r'CCLOADFLAGS = ..CCFLAGS.',
r'CCLOADFLAGS = $(CCFLAGS) -L' +
os.path.join(build_dir_path, 'lapack')],
]
elif compiler_type(test) == COMPILER_PGI:
replacements = [
]
elif compiler_type(test) == COMPILER_LLVM:
replacements = [
]
elif compiler_type(test) == COMPILER_XL:
replacements = [
]
else:
assert False, 'Compiler type not recognized. ' + compiler_type(test)
for replacement in replacements:
make_inc = re.sub(replacement[0], replacement[1], make_inc)
f = open(os.path.join(build_dir_path, 'scalapack', 'SLmake.inc'), 'w')
f.write(make_inc)
f.close()
make_command += ';' + get_module_command('list')
make_command += ';' + './make_scalapack.sh'
exit_status = os.system(make_command)
return exit_status
#------------------------------------------------------------------------------
def create_tmp_workspace(path1):
"""Create the workspace dir."""
os.makedirs(path1)
#------------------------------------------------------------------------------
if __name__ == '__main__':
main()
#------------------------------------------------------------------------------
#! /usr/bin/env python
"""
-------------------------------------------------------------------------------
File: check_executable.x
Author: Arnold Tharrington (arnoldt@ornl.gov)
Modified: Veronica G. Vergara Larrea, Wayne Joubert
National Center for Computational Sciences, Scientific Computing Group.
Oak Ridge National Laboratory
Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
-------------------------------------------------------------------------------
"""
import os
import argparse
import re
#------------------------------------------------------------------------------
IS_PASSING_YES = 1
IS_PASSING_NO = 0
#------------------------------------------------------------------------------
def process_command_line_args():
"""Get the command line arguments."""
command_description = (
'A program that checks the results located at <path_to_results>. '
'The check executable must write the status of the results to the '
'file Status/<test_id_string>/job_status.txt')
p_help = 'The absoulte path to the results of a test.'
i_help = 'The test id string.'
parser = argparse.ArgumentParser(description=command_description)
parser.add_argument('-p', help=p_help, required=True)
parser.add_argument('-i', help=i_help, required=True)
args = parser.parse_args()
return args
#------------------------------------------------------------------------------
def main():
"""Main program for check operation. Check the correctness of
the run results and report back.
"""
# Get the command line arguments.
args = process_command_line_args()
path_to_results = args.p
#test_id = args.i
# Compare the results.
is_passing = check_results(path_to_results)
# Write the status of the results to job data file.
write_to_job_status_file(path_to_results, is_passing)
#------------------------------------------------------------------------------
def check_results(path_to_results):
"""Perform the ciorrectness check of the results."""
# Make the file name paths to numbers squared.
file_path = os.path.join(path_to_results, "std.out.txt")
if not os.path.exists(file_path):
return IS_PASSING_NO
file_ = open(file_path, "r")
lines = file_.readlines()
file_.close()
num_total = None
num_passed = None
for line in lines:
match = re.findall('^Finished *(\d+) tests, '
'with the following results.*', line)
if len(match) == 1:
num_total = match
match = re.findall('^ *(\d+) tests completed and '
'passed residual checks.*', line)
if len(match) == 1:
num_passed = match
if num_total is None:
return IS_PASSING_NO
if num_passed is None:
return IS_PASSING_NO
return IS_PASSING_YES if num_passed == num_total else IS_PASSING_NO
#------------------------------------------------------------------------------
def write_to_job_status_file(path_to_results, is_passing):
"""Write the status of the results to job data file."""
# Get path.
dir_head1, dir_tail1 = os.path.split(path_to_results)
dir_head2, dir_tail2 = os.path.split(dir_head1)
file_path = os.path.join(dir_head2, 'Status', dir_tail1, 'job_status.txt')
file_ = open(file_path, 'w')
# Create the the string to write.
if is_passing == IS_PASSING_NO:
indicator = '1'
elif is_passing == IS_PASSING_YES:
indicator = '0'
elif is_passing >= 2:
indicator = '2'
string_ = '%s\n' % (indicator)
# Write the string.
file_.write(string_)
file_.close()
#------------------------------------------------------------------------------
if __name__ == '__main__':
main()
#------------------------------------------------------------------------------
#! /bin/bash -l
#------------------------------------------------------------------------------
#BSUB -q __batchqueue__
#BSUB -J __jobname__
#BSUB -o __resultsdir__/__jobname__.o%J
#BSUB -e __resultsdir__/__jobname__.e%J
#BSUB -n __nodes__
#BSUB -W __walltime__
#BSUB -b __starttime__
#-----------------------------------------------------
# Set up the environment for use of the harness.
#-----------------------------------------------------
source __rgtenvironmentalfile__
module load __nccstestharnessmodule__
module list
#-----------------------------------------------------
# Define some variables.
#-----------------------------------------------------
EXECUTABLE="__pathtoexecutable__"
STARTINGDIRECTORY="__startingdirectory__"
WORKDIR="__workdir__"
RESULTSDIR="__resultsdir__"
TEST_ID="__test_id__"
#-----------------------------------------------------
# Ensure that we are in the correct starting
# directory.
#-----------------------------------------------------
cd $STARTINGDIRECTORY
echo "Starting directory is $(pwd)"
#-----------------------------------------------------
# Make the working scratch space directory.
#-----------------------------------------------------
mkdir -p $WORKDIR
#-----------------------------------------------------
# Make the results directory if not already there.
#-----------------------------------------------------
#mkdir -p $RESULTSDIR
#-----------------------------------------------------
# Copy needed files to work dir.
#-----------------------------------------------------
EXECUTABLE_DIR=$(dirname $EXECUTABLE)
for FILE in $EXECUTABLE $EXECUTABLE_DIR"/"*.dat ; do
cp $FILE $WORKDIR
done
#-----------------------------------------------------
# Change directory to the working directory.
#-----------------------------------------------------
cd $WORKDIR
#-----------------------------------------------------
# Run the executable.
#-----------------------------------------------------
log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
--uniqueid $TEST_ID --mode start
__joblaunchcommand__
log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY \
--uniqueid $TEST_ID --mode final
sleep 30
#-----------------------------------------------------
# Ensure that we return to the starting directory.
#-----------------------------------------------------
cd $STARTINGDIRECTORY
#-----------------------------------------------------
# Copy the results back to the $RESULTSDIR.
#-----------------------------------------------------
cp -rf $WORKDIR"/"* $RESULTSDIR ### && rm -rf $WORKDIR
#-----------------------------------------------------
# Move the batch file to $RESULTSDIR.
#-----------------------------------------------------
if [ -e __batchfilename__ ] ; then
mv __batchfilename__ $RESULTSDIR
fi
#-----------------------------------------------------
# Check the final results.
#-----------------------------------------------------
check_executable_driver.py -p $RESULTSDIR -i $TEST_ID
#-----------------------------------------------------
# The script now determines if we are to resubmit
# itself.
#-----------------------------------------------------
case __resubmitme__ in
0)
test_harness_driver.py -r;;
1)
echo "No resubmit";;
esac
#------------------------------------------------------------------------------
#! /bin/bash -l
#------------------------------------------------------------------------------
#PBS -e __resultsdir__
#PBS -o __resultsdir__
#PBS -N __jobname__
#PBS -l walltime=__walltime__
#PBS -l nodes=__nodes__
#PBS -q __batchqueue__
#PBS -A __pbsaccountid__
#PBS -a __starttime__