Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Stansberry, Dale
spark_on_demand
Commits
db3b34f1
Commit
db3b34f1
authored
May 23, 2017
by
Stansberry, Dale
Browse files
- Fixes for Rhea spark module
parent
96525675
Changes
1
Hide whitespace changes
Inline
Side-by-side
spark_setup.py
View file @
db3b34f1
...
...
@@ -67,6 +67,7 @@ def generateScript():
outf
.
write
(
'source ${MODULESHOME}/init/bash
\n
'
)
outf
.
write
(
'module load python
\n
'
)
outf
.
write
(
'module load spark_on_demand/1.0.0
\n
'
)
outf
.
write
(
'export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
\n\n
'
)
outf
.
write
(
'if [ -d "$SPARK_DEPLOY_DIR/nodes" ]; then
\n
'
)
...
...
@@ -76,7 +77,7 @@ def generateScript():
outf
.
write
(
"echo =============== Running spark deployment ===============
\n\n
"
)
outf
.
write
(
'mpirun -n '
+
num_nodes
+
' --npernode 1 $SPARK_
HOME/sbin
/spark_deploy.py $SPARK_HOME $SPARK_DEPLOY_DIR &
\n\n
'
)
outf
.
write
(
'mpirun -n '
+
num_nodes
+
' --npernode 1 $SPARK_
ON_DEMAND_HOME
/spark_deploy.py $SPARK_HOME $SPARK_DEPLOY_DIR &
\n\n
'
)
outf
.
write
(
"MASTERFILE=$SPARK_DEPLOY_DIR/nodes/master
\n
"
)
outf
.
write
(
"timeout="
+
deploy_timeout
+
"
\n
"
)
outf
.
write
(
"while [ ! -f $MASTERFILE ]
\n
"
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment