buildscript

#!/bin/bash -ex
###############################################################################
# LINUX/MAC SCRIPT TO DRIVE THE JENKINS BUILDS OF MANTID.
#
# Notes:
#
# WORKSPACE, JOB_NAME, NODE_LABEL, PACKAGE_SUFFIX, GIT_COMMIT are
# environment variables that are set by Jenkins. The last one
# corresponds to any labels set on a slave.  BUILD_THREADS &
# PARAVIEW_DIR should be set in the configuration of each slave.
###############################################################################
SCRIPT_DIR=$(dirname "$0")

###############################################################################
# System discovery
###############################################################################
if [[ ${NODE_LABELS} == *rhel7* ]] || [[ ${NODE_LABELS} == *centos7* ]] || [[ ${NODE_LABELS} == *scilin7* ]]; then
  ON_RHEL7=true
fi
if [[ ${NODE_LABELS} == *ubuntu* ]]; then
  ON_UBUNTU=true
fi
if [[ ${NODE_LABELS} == *osx* ]]; then
  ON_MACOS=true
fi

###############################################################################
# All nodes currently have PARAVIEW_DIR and PARAVIEW_NEXT_DIR set
###############################################################################
export PARAVIEW_DIR=${PARAVIEW_DIR}

###############################################################################
# Print out the versions of things we are using
###############################################################################
# we use cmake3 on rhel because cmake is too old
if [ $(command -v cmake3) ]; then
    CMAKE_EXE=cmake3
    CPACK_EXE=cpack3
    CTEST_EXE=ctest3
else
    CMAKE_EXE=cmake
    CPACK_EXE=cpack
    CTEST_EXE=ctest
fi
$CMAKE_EXE --version

###############################################################################
# Check job requirements from the name and changes
###############################################################################
if [[ ${JOB_NAME} == *clean* ]]; then
  CLEANBUILD=true
fi
if [[ ${JOB_NAME} == *clang_tidy* ]]; then
  CLEANBUILD=true
fi

if [[ ${JOB_NAME} == *pull_requests* ]]; then
  PRBUILD=true
fi

if [[ ${JOB_NAME} == *debug* ]]; then
  BUILD_CONFIG="Debug"
elif [[ ${JOB_NAME} == *relwithdbg* ]]; then
  BUILD_CONFIG="RelWithDbg"
else
  BUILD_CONFIG="Release"
fi

# Setup software collections on rhel7 to allow using gcc7
if [[ $ON_RHEL7 ]]; then
  SCL_ENABLE="scl enable devtoolset-7"
else
  SCL_ENABLE="eval"
fi

# For pull requests decide on what to build based on changeset and Jenkins
# parameters.
DO_BUILD_CODE=true
DO_UNITTESTS=true
DO_DOCTESTS_USER=true
DO_BUILD_DEVDOCS=true
DO_BUILD_PKG=true
DO_SYSTEMTESTS=false
if [[ ${PRBUILD} == true ]]; then
  if ${SCRIPT_DIR}/check_for_changes dev-docs-only || ${SCRIPT_DIR}/check_for_changes user-docs-only; then
    DO_BUILD_CODE=false
    DO_UNITTESTS=false
  fi
  DO_DOCTESTS_USER=false
  DO_BUILD_DEVDOCS=false
  DO_BUILD_PKG=${BUILD_PACKAGE:-false}
  DO_SYSTEMTESTS=false

  if [[ ${ON_RHEL7} == true ]]; then
    # rhel does system testing
    if ! ${SCRIPT_DIR}/check_for_changes docs-gui-only; then
      DO_BUILD_PKG=true
      DO_SYSTEMTESTS=true
    fi
  elif [[ ${ON_UBUNTU} == true ]]; then
    # ubuntu does the docs build
    if ${SCRIPT_DIR}/check_for_changes dev-docs-only; then
      DO_BUILD_CODE=false
      DO_DOCTESTS_USER=false
    else
      DO_BUILD_CODE=true # code needs to be up to date
      DO_DOCTESTS_USER=true
    fi
  fi
fi

###############################################################################
# Setup the build directory
# For a clean build the entire thing is removed to guarantee it is clean. All
# other build types are assumed to be incremental and the following items
# are removed to ensure stale build objects don't interfere with each other:
#   - those removed by git clean -fdx --exclude=`basename $BUILD_DIR`
#   - build/bin/**: if libraries are removed from cmake they are not deleted
#                   from bin and can cause random failures
#   - build/ExternalData/**: data files will change over time and removing
#                            the links helps keep it fresh
###############################################################################
BUILD_DIR_REL=build
BUILD_DIR=$WORKSPACE/$BUILD_DIR_REL
if [ -z "$BUILD_DIR" ]; then
  echo "Build directory not set. Cannot continue"
  exit 1
fi

if [[ "$CLEANBUILD" == true ]]; then
  rm -rf $BUILD_DIR
fi
if [ -d $BUILD_DIR ]; then
  git clean -fdx --exclude=${BUILD_DIR_REL}
  rm -rf ${BUILD_DIR:?}/bin ${BUILD_DIR:?}/ExternalData
  find ${BUILD_DIR:?} -name 'TEST-*.xml' -delete
  if [[ -n ${CLEAN_EXTERNAL_PROJECTS} && "${CLEAN_EXTERNAL_PROJECTS}" == true ]]; then
      rm -rf $BUILD_DIR/eigen-*
      rm -rf $BUILD_DIR/googletest-*
      rm -fr $BUILD_DIR/python-xmlrunner-*
  fi
else
  mkdir $BUILD_DIR
fi

###############################################################################
# Setup clang
###############################################################################
if [[ ${JOB_NAME} == *clang* ]]; then
  USE_CLANG=true
elif [[ ${ON_MACOS} == true ]] ; then
  if [[ ! $(command -v icpc) ]] ; then
    USE_CLANG=true
  fi
fi

if [[ $USE_CLANG ]]; then
  # Assuming we are using the clang compiler
  echo "Using clang/llvm compiler."
  clang --version
  export CC=clang
  export CXX=clang++
  # check if this is also a clang-tidy build
  if [[ ${JOB_NAME} == *clang_tidy* ]]; then
      CLANGTIDYVAR="-DENABLE_CLANG_TIDY=ON"
  fi
  #check if CMakeCache.txt exists and if so that the cxx compiler is clang++
  #only needed with incremental builds. Clean builds delete this directory in a later step.
  if [[ -e $BUILD_DIR/CMakeCache.txt ]] && [[ ${JOB_NAME} != *clean* ]]; then
    COMPILERFILEPATH=$(grep 'CMAKE_CXX_COMPILER:FILEPATH' $BUILD_DIR/CMakeCache.txt)
    if [[ $COMPILERFILEPATH != *clang++* ]]; then
      # Removing the build directory entirely guarantees clang is used.
      rm -rf $BUILD_DIR
    fi
  fi
fi

#for openmp support on OS X run
# `brew install llvm`
# `ln -s /usr/local/opt/llvm/lib/libomp.dylib /usr/local/lib/libomp.dylib`
if [[ ${ON_MACOS} == true ]] ; then
  if [[ ${JOB_NAME} == *openmp* ]]; then
    export CC=/usr/local/opt/llvm/bin/clang
    export CXX=/usr/local/opt/llvm/bin/clang++
  fi
fi

###############################################################################
# Set up the location for the local object store outside of the build and
# source tree, which can be shared by multiple builds.
# It defaults to a MantidExternalData directory within the HOME directory.
# It can be overridden by setting the MANTID_DATA_STORE environment variable.
###############################################################################
if [ -z "$MANTID_DATA_STORE" ]; then
  export MANTID_DATA_STORE=$HOME/MantidExternalData
fi

###############################################################################
# Check if this is a Python 3 build and set CMake arguments.
###############################################################################
PY2_BUILD=false
PY3_BUILD=false
if [[ ${JOB_NAME} == *python3* ]]; then
  PY3_BUILD=true
  PYTHON3_EXECUTABLE=$(which python3)
  DIST_FLAGS="${DIST_FLAGS} -DPYTHON_EXECUTABLE=$PYTHON3_EXECUTABLE"
  PARAVIEW_DIR="${PARAVIEW_DIR}-python3"
else
  PY2_BUILD=true
fi

###############################################################################
# Packaging options
###############################################################################
if [[ ${DO_BUILD_PKG} == true ]]; then
  PACKAGINGVARS="-DPACKAGE_DOCS=ON"
  # Set some variables relating to the linux packages
  if [[ "${ON_MACOS}" == true ]]; then
    PACKAGINGVARS="${PACKAGINGVARS} -DCPACK_PACKAGE_SUFFIX="
  else
    # Use different suffix for linux builds if parameter is not defined
    if [[ -n "${PACKAGE_SUFFIX}" ]]; then
       echo "Using PACKAGE_SUFFIX=${PACKAGE_SUFFIX} from job parameter"
    elif [[ ${JOB_NAME} == *release* ]]; then
       PACKAGE_SUFFIX=""
    elif [[ ${JOB_NAME} == *master* ]]; then
      PACKAGE_SUFFIX="nightly"
    elif [[ ${JOB_NAME} == *pvnext* ]]; then
      PACKAGE_SUFFIX="mantidunstable-pvnext"
    else
      PACKAGE_SUFFIX="unstable"
    fi

    if [[ ${PY3_BUILD} == true ]]; then
      # Add '-python3' to package name and install path
      PACKAGE_SUFFIX=${PACKAGE_SUFFIX}-python3
    fi

    if [[ ${JOB_NAME} == *release* ]] && [[ ${PY2_BUILD} == true ]] && [[ -z "${PACKAGE_SUFFIX}" ]]; then
      # Traditional install path for python 2 release build
      PACKAGINGVARS="${PACKAGINGVARS} -DCMAKE_INSTALL_PREFIX=/opt/Mantid -DCPACK_PACKAGE_SUFFIX="
    else
      # everything else uses lower-case values
      PACKAGINGVARS="${PACKAGINGVARS} -DCMAKE_INSTALL_PREFIX=/opt/mantid${PACKAGE_SUFFIX} -DCPACK_PACKAGE_SUFFIX=${PACKAGE_SUFFIX}"
    fi

    if [[ ${ON_RHEL7} == true ]]; then
      if [[ -n "${RELEASE_NUMBER}" ]]; then
        RELEASE_NUMBER="1"
      fi
      PACKAGINGVARS="${PACKAGINGVARS} -DCPACK_RPM_PACKAGE_RELEASE=${RELEASE_NUMBER}"
    fi
  fi
fi


###############################################################################
# Generator
###############################################################################
if [ "$(command -v ninja)" ]; then
  CMAKE_GENERATOR="-G Ninja"
elif [ "$(command -v ninja-build)" ]; then
  CMAKE_GENERATOR="-G Ninja"
fi
if [ -e $BUILD_DIR/CMakeCache.txt ]; then
  CMAKE_GENERATOR=""
fi

###############################################################################
# Work in the build directory
###############################################################################
cd $BUILD_DIR

###############################################################################
# Clean up any artifacts from last build so that if it fails
# they don't get archived again
###############################################################################
rm -f -- *.dmg *.rpm *.deb *.tar.gz *.tar.xz

###############################################################################
# CMake configuration
###############################################################################
$SCL_ENABLE "${CMAKE_EXE} ${CMAKE_GENERATOR} -DCMAKE_BUILD_TYPE=${BUILD_CONFIG} -DENABLE_CPACK=ON -DMAKE_VATES=ON -DParaView_DIR=${PARAVIEW_DIR} -DMANTID_DATA_STORE=${MANTID_DATA_STORE} -DDOCS_HTML=ON -DENABLE_CONDA=ON -DENABLE_WORKBENCH=ON -DENABLE_FILE_LOGGING=OFF ${DIST_FLAGS} ${PACKAGINGVARS} ${CLANGTIDYVAR} .."

###############################################################################
# Coverity build should exit early
###############################################################################
if [[ ${JOB_NAME} == *coverity_build_and_submit* ]]; then
  ${COVERITY_DIR}/cov-build --dir cov-int ${CMAKE_EXE} --build . -- -j ${BUILD_THREADS:?}
  tar czvf mantid.tgz cov-int
  status=$(curl --form token=$COVERITY_TOKEN --form email=mantidproject@gmail.com \
       --form file=@mantid.tgz --form version=$GIT_COMMIT \
       https://scan.coverity.com/builds?project=mantidproject%2Fmantid)
  status=$(echo ${status} | sed -e 's/^ *//' -e 's/ *$//')
  if [[ -z $status ]] || [[ ${status} == "Build successfully submitted." ]]; then
    exit 0
  else
    echo "$status"
    exit 1
  fi
fi

###############################################################################
# Build step
###############################################################################
if [[ ${DO_BUILD_CODE} == true ]]; then
  ${CMAKE_EXE} --build . -- -j${BUILD_THREADS:?}
  ${CMAKE_EXE} --build . --target AllTests -- -j${BUILD_THREADS:?}
fi

###############################################################################
# Static analysis builds or stop here
###############################################################################
if [[ $USE_CLANG ]] && [[ ${JOB_NAME} == *clang_tidy* ]]; then
  exit 0
fi


###############################################################################
# Run the unit tests
###############################################################################
# Prevent race conditions when creating the user config directory
userconfig_dir=$HOME/.mantid
rm -fr $userconfig_dir
mkdir -p $userconfig_dir
touch $userconfig_dir/Mantid.user.properties

if [[ ${DO_UNITTESTS} == true ]]; then
  $CTEST_EXE -j${BUILD_THREADS:?} --schedule-random --output-on-failure
fi

###############################################################################
# User Documentation
###############################################################################
if [[ ${DO_DOCTESTS_USER} == true ]]; then
  # Remove doctrees directory so it forces a full reparse. It seems that
  # without this newly added doctests are not executed
  if [ -d $BUILD_DIR/docs/doctrees ]; then
    rm -fr $BUILD_DIR/docs/doctrees/*
  fi
  # Build HTML to verify that no referencing errors have crept in.
  ${CMAKE_EXE} --build . --target docs-html
  ${CMAKE_EXE} --build . --target docs-test
fi

###############################################################################
# Developer Documentation
###############################################################################
# Uncomment this when the dev-docs are ready to build without warnings
# if [[ ${DO_BUILD_DEVDOCS} == true ]]; then
#   if [ -d $BUILD_DIR/dev-docs/doctree ]; then
#     rm -fr $BUILD_DIR/dev-docs/doctree/*
#   fi
#   ${CMAKE_EXE} --build . --target dev-docs-html
# fi

###############################################################################
# Create the install kit if required. This includes building the Qt help
# documentation
###############################################################################
if [[ ${DO_BUILD_PKG} == true ]]; then
  # Workaround so that the target can find the properties file
  # CMake doesn't easily allow environment variables on custom targets
  if [[ ${ON_MACOS} == true ]]; then
    export MANTIDPATH=$PWD/bin
  fi
  ${CMAKE_EXE} --build . --target docs-qthelp
  ${CPACK_EXE}

  # Source tarball on clean build (arbitrarily choose rhel7)
  # Also, parcel up the documentation into a tar file that is easier to move around
  # and labelled by the commit id it was built with. This assumes the Jenkins git plugin
  # has set the GIT_COMMIT environment variable
  if [[ ${CLEANBUILD} == true && ${ON_RHEL7} == true ]]; then
    ${CMAKE_EXE} --build . --target docs-html
    tar -cjf mantiddocs-g${GIT_COMMIT:0:7}.tar.bz2 --exclude='*.buildinfo' --exclude="MantidProject.q*" docs/html
    # The ..._PREFIX argument avoids opt/Mantid directories at the top of the tree
    ${CPACK_EXE} --config CPackSourceConfig.cmake -D CPACK_PACKAGING_INSTALL_PREFIX=
  fi
fi

###############################################################################
# Run the system tests if required. Run from a package to have at least one
# Linux checks it install okay
###############################################################################
if [[ ${DO_SYSTEMTESTS} == true ]]; then
  if [[ ${PRBUILD} == true ]]; then
    EXTRA_ARGS="--exclude-in-pull-requests" $SCRIPT_DIR/systemtests
  else
    $SCRIPT_DIR/systemtests
  fi
fi