Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
olcf-acceptance-team
olcf4-acceptance-tests
Commits
03d9f52c
Commit
03d9f52c
authored
Aug 19, 2016
by
Joubert, Wayne
Browse files
Working code set up for ScaLAPACK testing.
parent
77ba4736
Changes
1000
Hide whitespace changes
Inline
Side-by-side
Too many changes to show.
To preserve performance only
20 of 1000+
files are displayed.
Plain diff
Email patch
Chester/ScaLAPACK
0 → 120000
View file @
03d9f52c
../Crest/ScaLAPACK
\ No newline at end of file
Crest/ScaLAPACK/Source/Common_Scripts/build_executable.x
0 → 100755
View file @
03d9f52c
#!/usr/bin/env python
"""
-------------------------------------------------------------------------------
File: build_executable.x
Author: Wayne Joubert (joubert@ornl.gov)
National Center for Computational Sciences, Scientific Computing Group.
Oak Ridge National Laboratory
Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
-------------------------------------------------------------------------------
"""
import
os
import
argparse
import
shutil
import
re
#------------------------------------------------------------------------------
def
process_command_line_args
():
"""Get the command line arguments."""
command_description
=
(
'A driver program that builds the binary for the test.'
)
p_help
=
(
'The absolute path to the workspace. This path must have the '
'appropiate permissions to permit the user of the test to r, w, and x.'
)
i_help
=
(
'The test id string. The build program uses this string to make a '
'unique directory within path_to_workspace. We don
\'
t want concurrent '
'builds to clobber each other. The submit program uses this string '
'to write the job schedule id to Status/<test_id_string>/job_id.txt'
)
parser
=
argparse
.
ArgumentParser
(
description
=
command_description
)
parser
.
add_argument
(
'-p'
,
help
=
p_help
,
required
=
True
)
parser
.
add_argument
(
'-i'
,
help
=
i_help
,
required
=
True
)
args
=
parser
.
parse_args
()
return
args
#------------------------------------------------------------------------------
def
main
():
"""Main program for building the executable."""
# Get the command line arguments.
args
=
process_command_line_args
()
path_to_workspace
=
args
.
p
#test_id = args.i
# Create the temporary workspace.
# Save the tempoary workspace for the submit executable.
create_tmp_workspace
(
path_to_workspace
)
# Make the binary.
prepare_to_make
(
path_to_workspace
)
exit_status
=
make_lapack
()
if
exit_status
!=
0
:
return
1
exit_status
=
make_binary
()
if
exit_status
!=
0
:
return
1
return
0
#------------------------------------------------------------------------------
def
prepare_to_make
(
path_to_workspace
):
"""Perform preparations, e.g., copying source tree."""
# Get the current working directory.
cwd
=
os
.
getcwd
()
# Get the 2 tail paths in the cwd.
dir_head1
,
dir_tail1
=
os
.
path
.
split
(
cwd
)
dir_head2
,
dir_tail2
=
os
.
path
.
split
(
dir_head1
)
# Get the path to the Source directory for the application.
path_to_source
=
os
.
path
.
join
(
dir_head2
,
'Source'
)
# Now make the path to the build directory.
build_dir_path
=
os
.
path
.
join
(
path_to_workspace
,
'build_directory'
)
# Copy Source to build directory.
print
(
'Copying source tree ...'
)
shutil
.
copytree
(
path_to_source
,
build_dir_path
,
symlinks
=
True
)
# Change to build directory.
os
.
chdir
(
build_dir_path
)
#------------------------------------------------------------------------------
def
is_cray
():
"""Helper: are we on a Cray system."""
return
'CRAYPE_VERSION'
in
os
.
environ
#------------------------------------------------------------------------------
COMPILER_GNU
=
'GNU'
COMPILER_PGI
=
'PGI'
COMPILER_CRAY
=
'CRAY'
COMPILER_INTEL
=
'INTEL'
COMPILER_LLVM
=
'LLVM'
COMPILER_XL
=
'XL'
#------------------------------------------------------------------------------
def
compiler_type
(
test
):
"""Helper: which compiler are we using."""
if
re
.
search
(
'_gnu'
,
test
)
!=
None
:
return
COMPILER_GNU
elif
re
.
search
(
'_pgi'
,
test
)
!=
None
:
return
COMPILER_PGI
elif
re
.
search
(
'_cray'
,
test
)
!=
None
:
return
COMPILER_CRAY
elif
re
.
search
(
'_intel'
,
test
)
!=
None
:
return
COMPILER_INTEL
elif
re
.
search
(
'_llvm'
,
test
)
!=
None
:
return
COMPILER_LLVM
elif
re
.
search
(
'_xl'
,
test
)
!=
None
:
return
COMPILER_XL
else
:
assert
False
,
'Compiler type not recognized. '
+
test
#------------------------------------------------------------------------------
def
get_module_command
(
module_command_args
):
"""Helper: et the shell code to do a module request."""
modules_home
=
os
.
environ
[
'MODULESHOME'
]
module_cmd
=
modules_home
+
'/bin/modulecmd'
return
'eval `'
+
module_cmd
+
' sh '
+
module_command_args
+
'`'
#------------------------------------------------------------------------------
def
make_lapack
():
"""Execute the make to build blas and lapack with test drivers."""
# Get dir names etc.
build_dir_path
=
os
.
getcwd
()
dir_head1
,
_
=
os
.
path
.
split
(
build_dir_path
)
dir_head2
,
_
=
os
.
path
.
split
(
dir_head1
)
_
,
test
=
os
.
path
.
split
(
dir_head2
)
# Modify makefile.
f
=
open
(
os
.
path
.
join
(
build_dir_path
,
'lapack'
,
'make.inc.example'
),
'r'
)
make_inc
=
f
.
read
()
f
.
close
()
make_command
=
'true'
if
is_cray
():
if
compiler_type
(
test
)
==
COMPILER_GNU
:
replacements
=
[
[
r
'FORTRAN = gfortran'
,
r
'FORTRAN = ftn'
],
[
r
'CC = gcc'
,
r
'CC = cc'
],
[
r
'OPTS = -O2'
,
r
'OPTS = -O2'
],
[
r
'NOOPT = -O0'
,
r
'NOOPT = -O0'
],
[
r
'LOADER = gfortran'
,
r
'LOADER = ftn'
],
[
r
'LOADOPTS ='
,
r
'LOADOPTS ='
],
[
r
'TIMER = INT_ETIME'
,
r
'TIMER = INT_ETIME'
],
]
make_command
+=
';'
+
get_module_command
(
'unload PrgEnv-pgi'
)
make_command
+=
';'
+
get_module_command
(
'load PrgEnv-gnu'
)
else
:
# IBM.
if
compiler_type
(
test
)
==
COMPILER_GNU
:
replacements
=
[
[
r
'FORTRAN = gfortran'
,
r
'FORTRAN = gfortran'
],
[
r
'CC = gcc'
,
r
'CC = gcc'
],
[
r
'OPTS = -O2'
,
r
'OPTS = -O2'
],
[
r
'NOOPT = -O0'
,
r
'NOOPT = -O0'
],
[
r
'LOADER = gfortran'
,
r
'LOADER = gfortran'
],
[
r
'LOADOPTS ='
,
r
'LOADOPTS ='
],
[
r
'TIMER = INT_ETIME'
,
r
'TIMER = INT_ETIME'
],
]
elif
compiler_type
(
test
)
==
'PGI'
:
replacements
=
[
]
elif
compiler_type
(
test
)
==
'LLVM'
:
replacements
=
[
]
elif
compiler_type
(
test
)
==
'XL'
:
replacements
=
[
]
else
:
assert
False
,
'Compiler type not recognized. '
+
compiler_type
(
test
)
for
replacement
in
replacements
:
make_inc
=
re
.
sub
(
replacement
[
0
],
replacement
[
1
],
make_inc
)
f
=
open
(
os
.
path
.
join
(
build_dir_path
,
'lapack'
,
'make.inc'
),
'w'
)
f
.
write
(
make_inc
)
f
.
close
()
make_command
+=
';'
+
get_module_command
(
'list'
)
make_command
+=
';'
+
'./make_lapack.sh'
exit_status
=
os
.
system
(
make_command
)
if
os
.
path
.
exists
(
os
.
path
.
join
(
build_dir_path
,
'lapack'
,
'librefblas.a'
)):
shutil
.
copyfile
(
os
.
path
.
join
(
build_dir_path
,
'lapack'
,
'librefblas.a'
),
os
.
path
.
join
(
build_dir_path
,
'lapack'
,
'libblas.a'
))
return
exit_status
#------------------------------------------------------------------------------
def
make_binary
():
"""Execute the make to build library and test drivers."""
# Get dir names etc.
build_dir_path
=
os
.
getcwd
()
dir_head1
,
_
=
os
.
path
.
split
(
build_dir_path
)
dir_head2
,
_
=
os
.
path
.
split
(
dir_head1
)
_
,
test
=
os
.
path
.
split
(
dir_head2
)
lapack_dir_path
=
os
.
path
.
join
(
build_dir_path
,
'lapack'
)
# Modify makefile.
f
=
open
(
os
.
path
.
join
(
build_dir_path
,
'scalapack'
,
'SLmake.inc.example'
),
'r'
)
make_inc
=
f
.
read
()
f
.
close
()
make_command
=
'true'
if
is_cray
():
if
compiler_type
(
test
)
==
COMPILER_GNU
:
replacements
=
[
[
r
'CDEFS = -DAdd_'
,
r
'CDEFS = -DAdd_'
],
[
r
'FC = mpif90'
,
r
'FC = ftn'
],
[
r
'CC = mpicc'
,
r
'CC = cc'
],
[
r
'FCFLAGS = -O3'
,
r
'FCFLAGS = -O3 -fPIC'
],
[
r
'CCFLAGS = -O3'
,
r
'CCFLAGS = -O3 -fPIC'
],
[
r
'FCLOADFLAGS = ..FCFLAGS.'
,
r
'FCLOADFLAGS = $(FCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'
],
[
r
'CCLOADFLAGS = ..CCFLAGS.'
,
r
'CCLOADFLAGS = $(CCFLAGS) -L$(CRAY_LIBSCI_PREFIX_DIR)/lib'
],
[
r
'BLASLIB = -lblas'
,
r
'BLASLIB = -lsci_gnu'
],
[
r
'LAPACKLIB = -llapack'
,
r
'LAPACKLIB = -lsci_gnu'
],
]
make_command
+=
';'
+
get_module_command
(
'unload PrgEnv-pgi'
)
make_command
+=
';'
+
get_module_command
(
'load PrgEnv-gnu'
)
make_command
+=
';'
+
get_module_command
(
'load cray-libsci'
)
elif
compiler_type
(
test
)
==
COMPILER_PGI
:
replacements
=
[
]
assert
False
,
'Compiler not yet implemented. '
+
compiler_type
(
test
)
elif
compiler_type
(
test
)
==
COMPILER_CRAY
:
replacements
=
[
]
assert
False
,
'Compiler not yet implemented. '
+
compiler_type
(
test
)
elif
compiler_type
(
test
)
==
COMPILER_INTEL
:
replacements
=
[
]
assert
False
,
'Compiler not yet implemented. '
+
compiler_type
(
test
)
else
:
assert
False
,
'Compiler type not recognized. '
+
compiler_type
(
test
)
else
:
# IBM.
if
compiler_type
(
test
)
==
COMPILER_GNU
:
replacements
=
[
[
r
'CDEFS = -DAdd_'
,
r
'CDEFS = -DAdd_'
],
[
r
'FC = mpif90'
,
r
'FC = mpfort'
],
[
r
'CC = mpicc'
,
r
'CC = mpcc'
],
[
r
'FCFLAGS = -O3'
,
r
'FCFLAGS = -O3 -compiler gnu'
],
[
r
'CCFLAGS = -O3'
,
r
'CCFLAGS = -O3 -compiler gnu'
],
[
r
'NOOPT = -O0'
,
r
'NOOPT = -O0 -compiler gnu'
],
[
r
'FCLOADFLAGS = ..FCFLAGS.'
,
r
'FCLOADFLAGS = $(FCFLAGS) -L'
+
os
.
path
.
join
(
build_dir_path
,
'lapack'
)],
[
r
'CCLOADFLAGS = ..CCFLAGS.'
,
r
'CCLOADFLAGS = $(CCFLAGS) -L'
+
os
.
path
.
join
(
build_dir_path
,
'lapack'
)],
]
elif
compiler_type
(
test
)
==
COMPILER_PGI
:
replacements
=
[
]
elif
compiler_type
(
test
)
==
COMPILER_LLVM
:
replacements
=
[
]
elif
compiler_type
(
test
)
==
COMPILER_XL
:
replacements
=
[
]
else
:
assert
False
,
'Compiler type not recognized. '
+
compiler_type
(
test
)
for
replacement
in
replacements
:
make_inc
=
re
.
sub
(
replacement
[
0
],
replacement
[
1
],
make_inc
)
f
=
open
(
os
.
path
.
join
(
build_dir_path
,
'scalapack'
,
'SLmake.inc'
),
'w'
)
f
.
write
(
make_inc
)
f
.
close
()
make_command
+=
';'
+
get_module_command
(
'list'
)
make_command
+=
';'
+
'./make_scalapack.sh'
exit_status
=
os
.
system
(
make_command
)
return
exit_status
#------------------------------------------------------------------------------
def
create_tmp_workspace
(
path1
):
"""Create the workspace dir."""
os
.
makedirs
(
path1
)
#------------------------------------------------------------------------------
if
__name__
==
'__main__'
:
main
()
#------------------------------------------------------------------------------
Crest/ScaLAPACK/Source/Common_Scripts/check_executable.x
0 → 100755
View file @
03d9f52c
#! /usr/bin/env python
"""
-------------------------------------------------------------------------------
File: check_executable.x
Author: Arnold Tharrington (arnoldt@ornl.gov)
Modified: Veronica G. Vergara Larrea, Wayne Joubert
National Center for Computational Sciences, Scientific Computing Group.
Oak Ridge National Laboratory
Copyright (C) 2016 Oak Ridge National Laboratory, UT-Battelle, LLC.
-------------------------------------------------------------------------------
"""
import
os
import
argparse
import
re
#------------------------------------------------------------------------------
IS_PASSING_YES
=
1
IS_PASSING_NO
=
0
#------------------------------------------------------------------------------
def
process_command_line_args
():
"""Get the command line arguments."""
command_description
=
(
'A program that checks the results located at <path_to_results>. '
'The check executable must write the status of the results to the '
'file Status/<test_id_string>/job_status.txt'
)
p_help
=
'The absoulte path to the results of a test.'
i_help
=
'The test id string.'
parser
=
argparse
.
ArgumentParser
(
description
=
command_description
)
parser
.
add_argument
(
'-p'
,
help
=
p_help
,
required
=
True
)
parser
.
add_argument
(
'-i'
,
help
=
i_help
,
required
=
True
)
args
=
parser
.
parse_args
()
return
args
#------------------------------------------------------------------------------
def
main
():
"""Main program for check operation. Check the correctness of
the run results and report back.
"""
# Get the command line arguments.
args
=
process_command_line_args
()
path_to_results
=
args
.
p
#test_id = args.i
# Compare the results.
is_passing
=
check_results
(
path_to_results
)
# Write the status of the results to job data file.
write_to_job_status_file
(
path_to_results
,
is_passing
)
#------------------------------------------------------------------------------
def
check_results
(
path_to_results
):
"""Perform the ciorrectness check of the results."""
# Make the file name paths to numbers squared.
file_path
=
os
.
path
.
join
(
path_to_results
,
"std.out.txt"
)
if
not
os
.
path
.
exists
(
file_path
):
return
IS_PASSING_NO
file_
=
open
(
file_path
,
"r"
)
lines
=
file_
.
readlines
()
file_
.
close
()
num_total
=
None
num_passed
=
None
for
line
in
lines
:
match
=
re
.
findall
(
'^Finished *(\d+) tests, '
'with the following results.*'
,
line
)
if
len
(
match
)
==
1
:
num_total
=
match
match
=
re
.
findall
(
'^ *(\d+) tests completed and '
'passed residual checks.*'
,
line
)
if
len
(
match
)
==
1
:
num_passed
=
match
if
num_total
is
None
:
return
IS_PASSING_NO
if
num_passed
is
None
:
return
IS_PASSING_NO
return
IS_PASSING_YES
if
num_passed
==
num_total
else
IS_PASSING_NO
#------------------------------------------------------------------------------
def
write_to_job_status_file
(
path_to_results
,
is_passing
):
"""Write the status of the results to job data file."""
# Get path.
dir_head1
,
dir_tail1
=
os
.
path
.
split
(
path_to_results
)
dir_head2
,
dir_tail2
=
os
.
path
.
split
(
dir_head1
)
file_path
=
os
.
path
.
join
(
dir_head2
,
'Status'
,
dir_tail1
,
'job_status.txt'
)
file_
=
open
(
file_path
,
'w'
)
# Create the the string to write.
if
is_passing
==
IS_PASSING_NO
:
indicator
=
'1'
elif
is_passing
==
IS_PASSING_YES
:
indicator
=
'0'
elif
is_passing
>=
2
:
indicator
=
'2'
string_
=
'%s
\n
'
%
(
indicator
)
# Write the string.
file_
.
write
(
string_
)
file_
.
close
()
#------------------------------------------------------------------------------
if
__name__
==
'__main__'
:
main
()
#------------------------------------------------------------------------------
Crest/ScaLAPACK/Source/Common_Scripts/lsf.template.x
0 → 100644
View file @
03d9f52c
#! /bin/bash -l
#------------------------------------------------------------------------------
#BSUB -q __batchqueue__
#BSUB -J __jobname__
#BSUB -o __resultsdir__/__jobname__.o%J
#BSUB -e __resultsdir__/__jobname__.e%J
#BSUB -n __nodes__
#BSUB -W __walltime__
#BSUB -b __starttime__
#-----------------------------------------------------
# Set up the environment for use of the harness.
#-----------------------------------------------------
source
__rgtenvironmentalfile__
module load __nccstestharnessmodule__
module list
#-----------------------------------------------------
# Define some variables.
#-----------------------------------------------------
EXECUTABLE
=
"__pathtoexecutable__"
STARTINGDIRECTORY
=
"__startingdirectory__"
WORKDIR
=
"__workdir__"
RESULTSDIR
=
"__resultsdir__"
TEST_ID
=
"__test_id__"
#-----------------------------------------------------
# Ensure that we are in the correct starting
# directory.
#-----------------------------------------------------
cd
$STARTINGDIRECTORY
echo
"Starting directory is
$(
pwd
)
"
#-----------------------------------------------------
# Make the working scratch space directory.
#-----------------------------------------------------
mkdir
-p
$WORKDIR
#-----------------------------------------------------
# Make the results directory if not already there.
#-----------------------------------------------------
#mkdir -p $RESULTSDIR
#-----------------------------------------------------
# Copy needed files to work dir.
#-----------------------------------------------------
EXECUTABLE_DIR
=
$(
dirname
$EXECUTABLE
)
for
FILE
in
$EXECUTABLE
$EXECUTABLE_DIR
"/"
*
.dat
;
do
cp
$FILE
$WORKDIR
done
#-----------------------------------------------------
# Change directory to the working directory.
#-----------------------------------------------------
cd
$WORKDIR
#-----------------------------------------------------
# Run the executable.
#-----------------------------------------------------
log_binary_execution_time.py
--scriptsdir
$STARTINGDIRECTORY
\
--uniqueid
$TEST_ID
--mode
start
__joblaunchcommand__
log_binary_execution_time.py
--scriptsdir
$STARTINGDIRECTORY
\
--uniqueid
$TEST_ID
--mode
final
sleep
30
#-----------------------------------------------------
# Ensure that we return to the starting directory.
#-----------------------------------------------------
cd
$STARTINGDIRECTORY
#-----------------------------------------------------
# Copy the results back to the $RESULTSDIR.
#-----------------------------------------------------
cp
-rf
$WORKDIR
"/"
*
$RESULTSDIR
### && rm -rf $WORKDIR
#-----------------------------------------------------
# Move the batch file to $RESULTSDIR.
#-----------------------------------------------------
if
[
-e
__batchfilename__
]
;
then
mv
__batchfilename__
$RESULTSDIR
fi
#-----------------------------------------------------
# Check the final results.
#-----------------------------------------------------
check_executable_driver.py
-p
$RESULTSDIR
-i
$TEST_ID
#-----------------------------------------------------
# The script now determines if we are to resubmit
# itself.
#-----------------------------------------------------
case
__resubmitme__
in
0
)
test_harness_driver.py
-r
;;
1
)
echo
"No resubmit"
;;
esac
#------------------------------------------------------------------------------
Crest/ScaLAPACK/Source/Common_Scripts/pbs.template.x
0 → 100644
View file @
03d9f52c
#! /bin/bash -l
#------------------------------------------------------------------------------
#PBS -e __resultsdir__
#PBS -o __resultsdir__
#PBS -N __jobname__
#PBS -l walltime=__walltime__
#PBS -l nodes=__nodes__
#PBS -q __batchqueue__
#PBS -A __pbsaccountid__