Commit 10936d7f authored by Hamilton, Steven P's avatar Hamilton, Steven P
Browse files

Adding longer running tests.

parent 0d7fd0d1
../Source/Correct_Results
\ No newline at end of file
../Source/Inputs
\ No newline at end of file
#!/usr/bin/env python3
import getopt
import sys
import os
import shutil
#import popen2
import subprocess
#
# Author: Arnold Tharrington
# Email: arnoldt@ornl.gov
# National Center of Computational Science, Scientifc Computing Group.
#
#
# This build the simple fortran program.
#
def main():
#
# Get the command line arguments.
#
try:
opts,args = getopt.getopt(sys.argv[1:],"hi:p:")
except getopt.GetoptError:
usage()
sys.exit(2)
#
# Parse the command line arguments.
#
if opts == []:
usage()
sys.exit()
for o, a in opts:
if o == "-p":
path_to_workspace = a
elif o == "-i":
test_id_string = a
elif o in ("-h", "--help"):
usage()
sys.exit()
else:
usage()
sys.exit()
#
# Create the temporary workspace.
# Save the tempoary workspace for the submit executable.
#
create_tmp_workspace(path_to_workspace)
#
#--Making the binary.
#
make_exit_status = make_binary(path_to_workspace)
if make_exit_status == 0:
make_exit_value = 0
else:
make_exit_value = 1
return make_exit_value
def make_binary(path_to_workspace):
#
# Get the current working directory.
#
cwd = os.getcwd()
#
# Get the 2 tail paths in the cwd.
#
(dir_head1, dir_tail1) = os.path.split(cwd)
(dir_head2, dir_tail2) = os.path.split(dir_head1)
#
# Get the path to the Source directory for the application.
#
path_to_source = os.path.join(dir_head2,"Source")
print(path_to_source)
#
# Now make the path to the build directory.
#
path_to_build_directory = os.path.join(path_to_workspace,"build_directory")
#
#Copy Source to build directory.
#
cmd1 = "cp -rf " + path_to_source + " " + path_to_build_directory
print(cmd1)
os.system(cmd1)
#
# Change back to build directory.
#
os.chdir(path_to_build_directory)
# Make executable.
cwd = os.getcwd()
print(cwd)
make_command = "./build_profugus.sh"
make_exit_status = os.system(make_command)
return make_exit_status
def usage():
print("Usage: build_executable.x [-h|--help] -p <path_to_worspace> -i <test_id_string>")
print("A driver program that the build the binary for the test.")
print()
print("-h, --help Prints usage information.")
print("-p The absolute path to the workspace. This path ")
print(" must have the appropiate permissions to permit ")
print(" the user of the test to r,w, and x. ")
print("-i The test id string. The build program ")
print(" uses this string to make a unique directory ")
print(" within path_to_workspace. We don't want ")
print(" concurrent builds to clobber each other. ")
def create_tmp_workspace(path1):
#
# Fisrt check to see if the path1 does not already exist.
#
os.makedirs(path1)
if __name__ == "__main__" :
main()
id="1"
path_to_workspace="/ccs/home/sphamil/Codes/Profugus/build/acceptance/${id}"
./build_executable.x -p ${path_to_workspace} -i ${id}
#! /usr/bin/env python3
import sys
import os
import getopt
import filecmp
import re
def check_executable_driver(path_to_results,
test_id_string):
#
# Compare the results.
#
jstatus = check_results(path_to_results)
#
# Write the statis of the results to job data file.
#
write_to_job_data(path_to_results,jstatus)
def get_path_to_correct_results():
cwd = os.getcwd()
#
# Get the head path in the cwd.
#
(dir_head1, dir_tail1) = os.path.split(cwd)
#
# This is the path to the correct results.
#
crslts = os.path.join(dir_head1,"Correct_Results")
return crslts
def check_results(path_to_results):
#-----------------------------------------------------
#Define good and bad results.
# -
#-----------------------------------------------------
GOOD_RESULTS=1
BAD_RESULTS=0
re_exp = re.compile("Total execution time")
#
# Make the file name paths to numbers squared.
#
file1 = os.path.join(path_to_results,"stdout.txt")
file_obj = open(file1,"r")
tlines = file_obj.readlines()
file_obj.close()
ip = 0
for record1 in tlines:
if re_exp.match(record1):
ip = ip+1;
expected_passes = 1
check_h5 = True
if check_h5:
expected_passes += 1
#
# Get path to the correct results.
#
path_to_correct_results = get_path_to_correct_results()
ref_outfile = "c5g7_3d_output_ref.h5"
ref_fluxfile = "c5g7_3d_flux_ref.h5"
outfile = "c5g7_3d_output.h5"
fluxfile = "c5g7_3d_flux.h5"
# Link reference files into results directory
if (not os.path.islink(os.path.join(path_to_results,ref_outfile))):
os.symlink(os.path.join(path_to_correct_results,ref_outfile),
os.path.join(path_to_results,ref_outfile))
if (not os.path.islink(os.path.join(path_to_results,ref_fluxfile))):
os.symlink(os.path.join(path_to_correct_results,ref_fluxfile),
os.path.join(path_to_results,ref_fluxfile))
import check_profugus
h5_result = check_profugus.check_solution(path_to_results)
if h5_result == 0:
ip += 1
if ip == expected_passes:
ival = GOOD_RESULTS
print("Passed")
else:
ival = BAD_RESULTS
print("Failed")
return ival
def write_to_job_data(path_to_results,jstatus):
(dir_head1, dir_tail1) = os.path.split(path_to_results)
(dir_head2, dir_tail2) = os.path.split(dir_head1)
file1 = os.path.join(dir_head2,"Status",dir_tail1,"job_status.txt")
file1_obj = open(file1,"w")
# Set the string to write to the job_status.txt file.
if jstatus == 0:
pf = "1"
elif jstatus == 1:
pf = "0"
elif jstatus >= 2:
pf = "2"
string1 = "%s\n" % (pf)
file1_obj.write(string1)
file1_obj.close()
def usage():
print("Usage: check_executable.x [-h|--help] [-i <test_id_string>] [-p <path_to_results>]")
print("A program that checks the results located at <path_to_results>")
print("The check executable must write the status of the results to the file")
print("Status/<test_id_string>/job_status.txt'.")
print("")
print("-h, --help Prints usage information.")
print("-p <path_to_results> The absoulte path to the results of a test.")
print("-i <test_id_string> The test string unique id.")
if __name__ == "__main__":
#
# Get the command line arguments.
#
try:
opts,args = getopt.getopt(sys.argv[1:],"hi:p:")
except getopt.GetoptError:
usage()
sys.exit(2)
#
# Parse the command line arguments.
#
for o, a in opts:
if o == "-p":
path_to_results = a
elif o == "-i":
test_id_string = a
elif o == ("-h", "--help"):
usage()
sys.exit()
else:
usage()
sys.exit()
check_executable_driver(path_to_results,
test_id_string)
id="1"
path_to_results="/lustre/atlas/scratch/sphamil/csc244/olcf4-acceptance-tests/summitdev/Profugus/CPU_1node/Run_Archive/${id}"
./check_executable.x -p ${path_to_results} -i ${id}
../../Source/Scripts/check_profugus.py
\ No newline at end of file
#! /bin/bash -l
#BSUB -q __batchqueue__
#BSUB -J __jobname__
#BSUB -o __resultsdir__/__jobname__.o%J
#BSUB -e __resultsdir__/__jobname__.e%J
#BSUB -nnodes __numbernodes__
#BSUB -W __walltime__
#BSUB -P __projectid__
#-----------------------------------------------------
# Set up the environment for use of the harness. -
# -
#-----------------------------------------------------
source __rgtenvironmentalfile__
module load __nccstestharnessmodule__
module load gcc/4.8.5
module load python/3.5.2
module load py-h5py/2.6.0-py3
module load spectrum-mpi
module list
#-----------------------------------------------------
# Define some variables. -
# -
#-----------------------------------------------------
EXECUTABLE="__pathtoexecutable__"
STARTINGDIRECTORY="__startingdirectory__"
WORKDIR="__workdir__"
RESULTSDIR="__resultsdir__"
UNIQUE_ID_STRING="__unique_id_string__"
INPUTDIR=${STARTINGDIRECTORY}/../Inputs
NUMPROCS="__numberprocessors__"
NUM_NODES="__numbernodes__"
echo "EXECUTABLE"
echo ${EXECUTABLE}
echo "WORKDIR"
echo ${WORKDIR}
echo "RESULTSDIR"
echo ${RESULTSDIR}
echo "INPUTDIR"
echo ${INPUTDIR}
echo "STARTINGDIRECTORY"
echo ${STARTINGDIRECTORY}
echo "NUMPROCS"
echo ${NUMPROCS}
echo "NUM_NODES"
echo ${NUM_NODES}
#-----------------------------------------------------
# Ensure that we are in the correct starting -
# directory. -
# -
#-----------------------------------------------------
cd $STARTINGDIRECTORY
#-----------------------------------------------------
# Make the working scratch space directory. -
# -
#-----------------------------------------------------
if [ ! -e $WORKDIR ]
then
mkdir -p $WORKDIR
fi
#-----------------------------------------------------
# Make the results directory. -
# -
#-----------------------------------------------------
if [ ! -e $RESULTSDIR ]
then
mkdir -p $RESULTSDIR
fi
#-----------------------------------------------------
# Change directory to the working directory. -
# -
#-----------------------------------------------------
cd $WORKDIR
echo "Changed to working directory"
pwd
ls -l
#-----------------------------------------------------
# Link input files into current directory
# -
#-----------------------------------------------------
echo "Contents of input directory:"
ls -l ${INPUTDIR}/
ln -s ${INPUTDIR}/c5g7_3d_cpu_1node_long.xml .
ln -s ${INPUTDIR}/c5g7_252g.xml .
#-----------------------------------------------------
# Run the executable. -
# -
#-----------------------------------------------------
log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY --uniqueid $UNIQUE_ID_STRING --mode start
__joblaunchcommand__
log_binary_execution_time.py --scriptsdir $STARTINGDIRECTORY --uniqueid $UNIQUE_ID_STRING --mode final
sleep 30
#-----------------------------------------------------
# Enusre that we return to the starting directory. -
# -
#-----------------------------------------------------
cd $STARTINGDIRECTORY
#-----------------------------------------------------
# Copy the results back to the $RESULTSDIR -
# -
#-----------------------------------------------------
cp -rf $WORKDIR/* $RESULTSDIR && rm -rf $WORKDIR
#-----------------------------------------------------
# Move the batch file name to $RESULTSDIR -
# -
#-----------------------------------------------------
mv __batchfilename__ $RESULTSDIR
#-----------------------------------------------------
# Check the final results. -
# -
#-----------------------------------------------------
check_executable_driver.py -p $RESULTSDIR -i $UNIQUE_ID_STRING
#-----------------------------------------------------
# The script now determines if we are to resubmit -
# itself. -
# -
#-----------------------------------------------------
case __resubmitme__ in
0)
test_harness_driver.py -r;;
1)
echo "No resubmit";;
esac
#!/usr/bin/env python
import os
import getopt
import sys
import re
import time
import subprocess
import shlex
#
# Author: Arnold Tharrington
# Email: arnoldt@ornl.gov
# National Center for Computational Sciences, Scientific Computing Group.
#
# Modified by: Veronica G. Vergara Larrea
# National Center for Computational Sciences, User Assistance Group.
def main():
#
# Get the command line arguments.
#
try:
opts,args = getopt.getopt(sys.argv[1:],"hi:p:r")
except getopt.GetoptError:
usage()
sys.exit(2)
#
# Parse the command line arguments.
#
if opts == []:
usage()
sys.exit()
#
# Initialize some variables.
#
batch_recursive_mode = "1"
for o, a in opts:
if o == "-p":
path_to_workspace = a
elif o == "-i":
test_id_string = a
elif o == "-r":
batch_recursive_mode = "0"
elif o in ("-h", "--help"):
usage()
sys.exit()
else:
usage()
sys.exit()
#
# Make the batch script.
#
scheduler = "lsf"
batchfilename = make_batch_script(batch_recursive_mode,path_to_workspace,test_id_string,scheduler)
#
# Submit the batch file to the scheduler.
#
sched_job_id = send_to_scheduler(batchfilename,scheduler)
print("Job id =" + str(sched_job_id))
#
#Write pbs job id to job_id.txt in the Status dir.
#
write_job_id_to_status(sched_job_id,test_id_string)
def make_batch_script(batch_recursive_mode,path_to_workspace,test_id_string,scheduler):
#
# Define the batch file names.
#
if scheduler == "pbs":
print("Using PBS scheduler syntax")
batchtemplatefilename = "pbs.template.x"
sched_id = 0
elif scheduler == "lsf":
print("Using LSF scheduler syntax")
batchtemplatefilename = "lsf.template.x"
sched_id = 1
else:
print("Scheduler " + scheduler + " is not supported")
sched_id = -1
batchfilename = "run_profugus.sh"
#
# Define the parse definitons and the regular expressions.
#
nccstestharnessmodule = os.environ["RGT_NCCS_TEST_HARNESS_MODULE"]
rgtenvironmentalfile = os.environ["RGT_ENVIRONMENTAL_FILE"]
jobname = "profugus_cpu_1node"
nodes = "1"
ppn = "160"
ranks = str(int(nodes) * int(ppn))
batchqueue = "batch"
pbsaccountid = os.environ["RGT_PBS_JOB_ACCNT_ID"]
pathtoexecutable = os.path.join(path_to_workspace,"build_directory","bin/xmc")
startingdirectory = os.getcwd()
resultsdir = get_path_to_results_dir(test_id_string)
workdir = os.path.join(path_to_workspace,"workdir")
resubmitme = batch_recursive_mode
walltime = "04:00"
joblaunchcommand = "jsrun --nrs ${NUM_NODES} --cpu_per_rs 20 --tasks_per_rs 160 --rs_per_host 1 -E LD_LIBRARY_PATH ${EXECUTABLE} -i c5g7_3d_cpu_1node_long.xml | tee stdout.txt"
rg_array = [
(re.compile("__jobname__"),jobname),
(re.compile("__walltime__"),walltime),
(re.compile("__numberprocessors__"),ranks),
(re.compile("__numbernodes__"),nodes),
(re.compile("__nccstestharnessmodule__"),nccstestharnessmodule),
(re.compile("__rgtenvironmentalfile__"),rgtenvironmentalfile),
(re.compile("__batchqueue__"),batchqueue),
(re.compile("__projectid__"),pbsaccountid),
(re.compile("__pathtoexecutable__"),pathtoexecutable),
(re.compile("__startingdirectory__"),startingdirectory),
(re.compile("__resultsdir__"),resultsdir),
(re.compile("__workdir__"),workdir),
(re.compile("__joblaunchcommand__"),joblaunchcommand),
(re.compile("__resubmitme__"),resubmitme),
(re.compile("__unique_id_string__"),test_id_string),
(re.compile("__batchfilename__"),batchfilename),
]
#
# Read the lines of the batch template file.
#
templatefileobject = open(batchtemplatefilename,"r")
tlines = templatefileobject.readlines()
templatefileobject.close()
#
# Here is where we actually make the pbs batch file from pbs.template.x.
#
fileobject = open(batchfilename,"w")
for record1 in tlines:
for (regexp,text1) in rg_array:
record1 = regexp.sub(text1,record1)