Merge pull request #256 from scottwittenburg/more-image-improvements

CI improvements

.circleci/config.yml

@@ -3,15 +3,18 @@ defaults: &defaults
   steps:
     - checkout:
         path: /home/adios2/source
+    - run:
+        name: Update
+        command: bash source/scripts/circle/runOnCircle.sh update
     - run:
         name: Configure
-        command: bash --login source/scripts/circle/runOnCircle.sh configure
+        command: bash source/scripts/circle/runOnCircle.sh configure
     - run:
         name: Build
-        command: bash --login source/scripts/circle/runOnCircle.sh build
+        command: bash source/scripts/circle/runOnCircle.sh build
     - run:
         name: Test
-        command: bash --login source/scripts/circle/runOnCircle.sh test
+        command: bash source/scripts/circle/runOnCircle.sh test
 
 version: 2
 

scripts/circle/.dockerignore

 runOnCircle.sh
 postComment.sh
 *.cmake
+findStatus.py

scripts/circle/Dockerfile

@@ -13,8 +13,7 @@ RUN yum -y install \
 
 # Install gcc7 OpenHPC packages
 RUN yum -y install \
-      lmod-ohpc gnu7-compilers-ohpc openmpi-gnu7-ohpc phdf5-gnu7-openmpi-ohpc \
-      python-numpy-gnu7-ohpc
+      lmod-ohpc gnu7-compilers-ohpc openmpi-gnu7-ohpc phdf5-gnu7-openmpi-ohpc
 
 # Cleanup headers and packages
 RUN yum clean all
@@ -63,14 +62,23 @@ RUN cd source && \
 # Cleanup
 RUN rm -rf /opt/adios1/1.12.0/source
 
+# mpi4py
+WORKDIR /opt
+RUN . /etc/profile && \
+    module purge && \
+    module load gnu7 openmpi && \
+    curl -L https://bitbucket.org/mpi4py/mpi4py/downloads/mpi4py-2.0.0.tar.gz | \
+      tar -xzv && \
+    cd mpi4py-2.0.0 && \
+    python setup.py build && \
+    python setup.py install && \
+    cd ../ && rm -rf mpi4py-2.0.0
+
 # Create a non-root user to run the builds/tests
 RUN export uid=1001 gid=1001 && \
     mkdir -p /home/adios2 && \
     echo "adios2:x:${uid}:${gid}:adios2,,,:/home/adios2:/bin/bash" >> /etc/passwd && \
     echo "adios2:x:${uid}:" >> /etc/group && \
-    mkdir -p /etc/sudoers.d && \
-    echo "adios2 ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/adios2 && \
-    chmod 0440 /etc/sudoers.d/adios2 && \
     chown ${uid}:${gid} -R /home/adios2
 
 USER adios2

scripts/circle/circle_el7-gcc48.cmake

 # Client maintainer: chuck.atkins@kitware.com
 set(CTEST_SITE "CircleCI")
-set(CTEST_BUILD_NAME "$ENV{CIRCLE_BRANCH}_$ENV{CIRCLE_JOB}")
 set(CTEST_BUILD_CONFIGURATION Release)
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
 set(CTEST_TEST_ARGS PARALLEL_LEVEL 4)
 
 set(dashboard_model Experimental)
-set(dashboard_binary_name "build-$ENV{CIRCLE_JOB}")
+set(dashboard_binary_name "build_$ENV{CIRCLE_JOB}")
 
+set(CTEST_GIT_COMMAND "/usr/bin/git")
+set(CTEST_UPDATE_VERSION_ONLY TRUE)
 set(CTEST_SOURCE_DIRECTORY "$ENV{CIRCLE_WORKING_DIRECTORY}/source")
 set(CTEST_DASHBOARD_ROOT "$ENV{HOME}")
 

scripts/circle/circle_el7-gcc7-openmpi.cmake

 # Client maintainer: chuck.atkins@kitware.com
 set(CTEST_SITE "CircleCI")
-set(CTEST_BUILD_NAME "$ENV{CIRCLE_BRANCH}_$ENV{CIRCLE_JOB}")
 set(CTEST_BUILD_CONFIGURATION Release)
 set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
 set(CTEST_BUILD_FLAGS "-k -j4")
@@ -9,6 +8,8 @@ set(CTEST_TEST_ARGS PARALLEL_LEVEL 4)
 set(dashboard_model Experimental)
 set(dashboard_binary_name "build_$ENV{CIRCLE_JOB}")
 
+set(CTEST_GIT_COMMAND "/usr/bin/git")
+set(CTEST_UPDATE_VERSION_ONLY TRUE)
 set(CTEST_SOURCE_DIRECTORY "$ENV{CIRCLE_WORKING_DIRECTORY}/source")
 set(CTEST_DASHBOARD_ROOT "$ENV{HOME}")
 
@@ -17,7 +18,6 @@ module(purge)
 module(load gnu7)
 module(load openmpi)
 module(load phdf5)
-module(load python)
 
 set(ENV{CC}  gcc)
 set(ENV{CXX} g++)
@@ -30,7 +30,7 @@ ADIOS2_USE_DataMan:STRING=ON
 ADIOS2_USE_Fortran:STRING=ON
 ADIOS2_USE_HDF5:STRING=ON
 ADIOS2_USE_MPI:STRING=ON
-ADIOS2_USE_Python:STRING=OFF
+ADIOS2_USE_Python:STRING=ON
 ADIOS2_USE_ZFP:STRING=ON
 ADIOS2_USE_ZeroMQ:STRING=ON
 ZFP_ROOT_DIR:PATH=/opt/zfp/install

scripts/circle/findStatus.py

+import sys
+import json
+import argparse
+
+
+def searchForContext(context):
+    print('Searching for a status with context: %s' % context)
+    statuses = json.load(sys.stdin)
+
+    for stat in statuses:
+        if 'context' in stat and stat['context'] == context:
+            sys.exit(0)
+
+    sys.exit(1)
+
+
+# =============================================================================
+# Main
+# =============================================================================
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Parse github api status list")
+    parser.add_argument("--context", default=None, help="context of interest")
+    args = parser.parse_args()
+    searchForContext(args.context)

scripts/circle/postComment.sh

@@ -37,3 +37,4 @@ for i in "${ADDR[@]}"; do
       echo "Unable to find PR number in ${i}"
     fi
 done
+

scripts/circle/runOnCircle.sh

 #!/bin/bash
 
+API_BASE="https://api.github.com/repos/ornladios/adios2"
+USER=${STATUS_ROBOT_NAME}
+TOKEN=${STATUS_ROBOT_KEY}
+COMMIT=${CIRCLE_SHA1}
+CDASH_STATUS_CONTEXT="cdash-results-link"
+
+build_status_body() {
+  cat <<EOF
+{
+  "state": "success",
+  "target_url": "https://open.cdash.org/index.php?compare1=61&filtercount=1&field1=revision&project=ADIOS&showfilters=0&limit=100&value1=${COMMIT}&showfeed=0",
+  "description": "Build and test results can also be viewed on CDash",
+  "context": "${CDASH_STATUS_CONTEXT}"
+}
+EOF
+}
+
+check_and_post_status() {
+  PYTHON_SCRIPT="${SOURCE_DIR}/scripts/circle/findStatus.py"
+  curl -u "${STATUS_ROBOT_NAME}:${STATUS_ROBOT_KEY}" "${API_BASE}/commits/${COMMIT}/statuses" | python ${PYTHON_SCRIPT} --context ${CDASH_STATUS_CONTEXT}
+  if [ $? -ne 0 ]
+  then
+    echo "Need to post a status for context ${CDASH_STATUS_CONTEXT}"
+    postBody="$(build_status_body)"
+    postUrl="${API_BASE}/statuses/${COMMIT}"
+    curl -u "${OWNER}:${TOKEN}" "${postUrl}" -H "Content-Type: application/json" -H "Accept: application/vnd.github.v3+json" -d "${postBody}"
+  fi
+}
+
+get_real_branch_name() {
+  APIURL="${API_BASE}/pulls/${CIRCLE_PR_NUMBER}"
+  RESULT=`curl -s ${APIURL} | python -c "import sys, json; print(json.load(sys.stdin)['head']['ref'])" 2> /dev/null`
+
+  if [ $? -eq 0 ]
+  then
+    REALBRANCH=$RESULT
+  else
+    REALBRANCH=$CIRCLE_BRANCH
+  fi
+}
+
 check_var() {
   if [ -z "$1" ]
   then
@@ -11,6 +52,21 @@ check_var() {
 check_var CIRCLE_WORKING_DIRECTORY
 check_var CIRCLE_BRANCH
 check_var CIRCLE_JOB
+check_var CIRCLE_BUILD_NUM
+check_var CIRCLE_PR_NUMBER
+
+if [ ! "${CUSTOM_BUILD_NAME}" ]
+then
+  get_real_branch_name
+
+  LINETOSAVE="export CUSTOM_BUILD_NAME=${REALBRANCH}_${CIRCLE_BUILD_NUM}_${CIRCLE_JOB}"
+
+  # Set the custom build name for this step
+  eval $LINETOSAVE
+
+  # Also make sure it will get set for the following steps
+  echo "${LINETOSAVE}" >> $BASH_ENV
+fi
 
 SOURCE_DIR=${CIRCLE_WORKING_DIRECTORY}/source
 CTEST_SCRIPT="${SOURCE_DIR}/scripts/circle/circle_${CIRCLE_JOB}.cmake"
@@ -22,13 +78,20 @@ then
 fi
 
 case "$1" in
-  configure|build|test)
+  update|configure|build|test)
     STEP=$1
+    if [ "$STEP" == "update" ]
+    then
+      check_and_post_status
+    fi
     ;;
   *)
-    echo "Usage: $0 (configure|build|test)"
+    echo "Usage: $0 (update|configure|build|test)"
     exit 3
     ;;
 esac
 
-/opt/cmake/3.6.0/bin/ctest -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF -Ddashboard_do_${STEP}=TRUE
+# Manually source the bash env setup, freeing up $BASH_ENV used by circleci
+. /etc/profile >/dev/null
+
+/opt/cmake/3.6.0/bin/ctest -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF -Ddashboard_do_${STEP}=TRUE -DCTEST_BUILD_NAME=${CUSTOM_BUILD_NAME}

scripts/dashboard/adios_common.cmake

@@ -32,6 +32,7 @@
 #   dashboard_cache           = Initial CMakeCache.txt file content
 
 #   dashboard_do_checkout  = True to enable source checkout via git
+#   dashboard_do_update    = True to enable source update
 #   dashboard_do_configure = True to enable the Configure step
 #   dashboard_do_build     = True to enable the Build step
 #   dashboard_do_test      = True to enable the Test step

scripts/dashboard/common.cmake

@@ -33,6 +33,7 @@
 #   dashboard_cache       = Initial CMakeCache.txt file content
 
 #   dashboard_do_checkout  = True to enable source checkout via git
+#   dashboard_do_update    = True to enable the Update step
 #   dashboard_do_configure = True to enable the Configure step
 #   dashboard_do_build     = True to enable the Build step
 #   dashboard_do_test      = True to enable the Test step
@@ -68,6 +69,10 @@ if(NOT DEFINED dashboard_full)
 endif()
 
 # Initialize all build steps to "ON"
+if(NOT DEFINED dashboard_do_update)
+  set(dashboard_do_update ${dashboard_full})
+endif()
+
 if(NOT DEFINED dashboard_do_checkout)
   set(dashboard_do_checkout ${dashboard_full})
 endif()
@@ -94,7 +99,7 @@ if(NOT DEFINED dashboard_do_memcheck)
 endif()
 
 if(NOT DEFINED dashboard_fresh)
-  if(dashboard_full OR dashboard_do_configure)
+  if(dashboard_full OR dashboard_do_update)
     set(dashboard_fresh TRUE)
   else()
     set(dashboard_fresh FALSE)
@@ -368,6 +373,18 @@ else()
   ctest_start(${dashboard_model} APPEND)
 endif()
 
+# Look for updates.
+if(dashboard_do_update)
+  if(COMMAND dashboard_hook_update)
+    dashboard_hook_update()
+  endif()
+  message("Calling ctest_update...")
+  ctest_update(RETURN_VALUE count)
+  set(CTEST_CHECKOUT_COMMAND) # checkout on first iteration only
+  message("Found ${count} changed files")
+  ctest_submit(PARTS Update)
+endif()
+
 if(dashboard_do_configure)
   if(COMMAND dashboard_hook_configure)
     dashboard_hook_configure()