buildscript 18.3 KB
Newer Older
1
#!/bin/bash -ex
2
###############################################################################
3
# LINUX/MAC SCRIPT TO DRIVE THE JENKINS BUILDS OF MANTID.
4
5
6
#
# Notes:
#
7
8
# WORKSPACE, JOB_NAME, NODE_LABEL, PACKAGE_SUFFIX, GIT_COMMIT are
# environment variables that are set by Jenkins. The last one
9
# corresponds to any labels set on a slave.  BUILD_THREADS should
10
# be set in the configuration of each slave.
11
###############################################################################
12
13
14
# Set all string comparisons to case insensitive (i.e. Release == release)
shopt -s nocasematch

15
SCRIPT_DIR=$(dirname "$0")
16
17
18
19
20
21
22
23
XVFB_SERVER_NUM=101
ULIMIT_CORE_ORIG=$(ulimit -c)

###############################################################################
# Functions
###############################################################################

function onexit {
24
25
26
    if [[ ${USE_CORE_DUMPS} == true ]]; then
        ulimit -c $ULIMIT_CORE_ORIG
    fi
27
28
29
}

function run_with_xvfb {
30
    if [ $(command -v xvfb-run) ]; then
31
32
33
        # Use -e because a bug on RHEL7 means --error-file produces an error: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=337703;msg=2
        # Use -noreset because of an X Display bug caused by a race condition in xvfb: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1102
        xvfb-run -e /dev/stderr --server-args="-core -noreset -screen 0 640x480x24" \
34
35
36
37
        --server-num=${XVFB_SERVER_NUM} $@
    else
        eval $@
    fi
38
39
}

40
41
42
43
44
45
46
47
48
49
50
function terminate_xvfb_sessions() {
    if [ $(command -v xvfb-run) ]; then
        echo "Terminating existing Xvfb sessions"

        # Kill Xvfb processes
        killall Xvfb || true

        # Remove Xvfb X server lock files
        rm -f /tmp/.X${XVFB_SERVER_NUM}-lock
    fi
}
51

52
53
54
###############################################################################
# Terminate existing Xvfb sessions
###############################################################################
55
terminate_xvfb_sessions
56

57
58
59
###############################################################################
# System discovery
###############################################################################
60
USE_CORE_DUMPS=true
61
if [[ ${NODE_LABELS} == *rhel7* ]] || [[ ${NODE_LABELS} == *centos7* ]] || [[ ${NODE_LABELS} == *scilin7* ]]; then
62
63
64
65
66
67
    ON_RHEL7=true
    elif [[ ${NODE_LABELS} == *ubuntu* ]]; then
    ON_UBUNTU=true
    elif [[ ${NODE_LABELS} == *osx* ]]; then
    ON_MACOS=true
    USE_CORE_DUMPS=false
68
fi
69

70
71
72
73
74
75
76
77
# Setup software collections on rhel7 to allow using gcc7
if [[ $ON_RHEL7 ]]; then
    SCL_ENABLE="scl enable devtoolset-7"
else
    SCL_ENABLE="eval"
fi


78
79
80
###############################################################################
# Script cleanup
###############################################################################
81
trap onexit INT TERM EXIT
82

83
84
85
86
87
88
89
90
91
92
93
###############################################################################
# Some preprocessing steps on the workspace
###############################################################################
BUILD_DIR_REL=build
BUILD_DIR=$WORKSPACE/$BUILD_DIR_REL

# Clean the source tree to remove stale configured files but make sure to
# leave external/ and the BUILD_DIR directory intact.
# There is a later check to see if this is a clean build and remove BUILD_DIR.
git clean -d -x --force --exclude=${BUILD_DIR_REL} --exclude=".Xauthority-*"

94
95
96
###############################################################################
# Print out the versions of things we are using
###############################################################################
Peterson, Peter's avatar
Peterson, Peter committed
97
# we use cmake3 on rhel because cmake is too old
98
99
if [ $(command -v cmake3) ]; then
    CMAKE_EXE=cmake3
Peterson, Peter's avatar
Peterson, Peter committed
100
101
    CPACK_EXE=cpack3
    CTEST_EXE=ctest3
102
103
else
    CMAKE_EXE=cmake
Peterson, Peter's avatar
Peterson, Peter committed
104
105
    CPACK_EXE=cpack
    CTEST_EXE=ctest
106
107
fi
$CMAKE_EXE --version
108

109
###############################################################################
110
# Check job requirements from the name and changes
111
###############################################################################
112
if [[ ${JOB_NAME} == *clean* || ${JOB_NAME} == *clang_tidy* ]]; then
113
    CLEANBUILD=true
114
fi
115

116
if [[ ${JOB_NAME} == *pull_requests* ]]; then
117
    PRBUILD=true
118
119
fi

120
if [[ ${JOB_NAME} == *debug* ]]; then
121
122
123
    BUILD_CONFIG="Debug"
    elif [[ ${JOB_NAME} == *relwithdbg* ]]; then
    BUILD_CONFIG="RelWithDbg"
124
else
125
    BUILD_CONFIG="Release"
126
127
fi

128
129
# Setup software collections on rhel7 to allow using gcc7
if [[ $ON_RHEL7 ]]; then
130
    SCL_ENABLE="scl enable devtoolset-7"
131
else
132
    SCL_ENABLE="eval"
133
134
fi

135
136
# For pull requests decide on what to build based on changeset and Jenkins
# parameters.
137
138
139
140
141
142
DO_BUILD_CODE=true
DO_UNITTESTS=true
DO_DOCTESTS_USER=true
DO_BUILD_DEVDOCS=true
DO_BUILD_PKG=true
DO_SYSTEMTESTS=false
143

144
if [[ ${PRBUILD} == true ]]; then
145
146
147
    if ${SCRIPT_DIR}/check_for_changes dev-docs-only || ${SCRIPT_DIR}/check_for_changes user-docs-only; then
        DO_BUILD_CODE=false
        DO_UNITTESTS=false
148
    fi
149
150
151
152
    DO_DOCTESTS_USER=false
    DO_BUILD_DEVDOCS=false
    DO_BUILD_PKG=${BUILD_PACKAGE:-false}
    DO_SYSTEMTESTS=false
153

154
    if [[ ${ON_RHEL7} == true ]]; then
155
        # rhel does system testing if there are any non-doc or gui changes
156
        if ! ${SCRIPT_DIR}/check_for_changes docs-gui-only; then
157
158
            DO_BUILD_PKG=true
            DO_SYSTEMTESTS=true
159
160
161
162
163
164
165
166
167
168
        fi
        elif [[ ${ON_UBUNTU} == true ]]; then
        # ubuntu does the docs build
        if ${SCRIPT_DIR}/check_for_changes dev-docs-only; then
            DO_BUILD_CODE=false
            DO_DOCTESTS_USER=false
        else
            DO_BUILD_CODE=true # code needs to be up to date
            DO_DOCTESTS_USER=true
        fi
169
    fi
170
171
fi

172
173
174
175
176
177
178
179
180
###############################################################################
# Setup the build directory
# For a clean build the entire thing is removed to guarantee it is clean. All
# other build types are assumed to be incremental and the following items
# are removed to ensure stale build objects don't interfere with each other:
#   - build/bin/**: if libraries are removed from cmake they are not deleted
#                   from bin and can cause random failures
#   - build/ExternalData/**: data files will change over time and removing
#                            the links helps keep it fresh
181
182
#   - build/Testing/**: old ctest xml files will change over time and removing
#                       the links helps keep it fresh
183
184
###############################################################################
if [ -z "$BUILD_DIR" ]; then
185
186
    echo "Build directory not set. Cannot continue"
    exit 1
187
188
189
fi

if [[ "$CLEANBUILD" == true ]]; then
190
    rm -rf $BUILD_DIR
191
fi
192
193
194
195
196

mkdir -p $BUILD_DIR

# Tidy build dir
rm -rf ${BUILD_DIR:?}/bin ${BUILD_DIR:?}/ExternalData ${BUILD_DIR:?}/Testing
197
find ${BUILD_DIR:?} \( -name 'TEST-*.xml' -o -name 'Test.xml' \) -delete
198
199
200
201

if [[ -n ${CLEAN_EXTERNAL_PROJECTS} && "${CLEAN_EXTERNAL_PROJECTS}" == true ]]; then
    rm -rf $BUILD_DIR/eigen-*
    rm -rf $BUILD_DIR/googletest-*
202
203
fi

204
205
206
###############################################################################
# Setup clang
###############################################################################
207
if [[ ${JOB_NAME} == *clang* ]]; then
208
    USE_CLANG=true
209
210
211
212
    elif [[ ${ON_MACOS} == true ]] ; then
    if [[ ! $(command -v icpc) ]] ; then
        USE_CLANG=true
    fi
213
fi
214

215
if [[ $USE_CLANG ]]; then
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    # Assuming we are using the clang compiler
    echo "Using clang/llvm compiler."
    clang --version
    export CC=clang
    export CXX=clang++
    # check if this is also a clang-tidy build
    if [[ ${JOB_NAME} == *clang_tidy* ]]; then
        CLANGTIDYVAR="-DENABLE_CLANG_TIDY=ON"
    fi
    #check if CMakeCache.txt exists and if so that the cxx compiler is clang++
    #only needed with incremental builds. Clean builds delete this directory in a later step.
    if [[ -e $BUILD_DIR/CMakeCache.txt ]] && [[ ${JOB_NAME} != *clean* ]]; then
        COMPILERFILEPATH=$(grep 'CMAKE_CXX_COMPILER:FILEPATH' $BUILD_DIR/CMakeCache.txt)
        if [[ $COMPILERFILEPATH != *clang++* ]]; then
            # Removing the build directory entirely guarantees clang is used.
            rm -rf $BUILD_DIR
        fi
233
    fi
234
235
fi

236
237
238
#for openmp support on OS X run
# `brew install llvm`
# `ln -s /usr/local/opt/llvm/lib/libomp.dylib /usr/local/lib/libomp.dylib`
239
if [[ ${ON_MACOS} == true ]] ; then
240
241
242
243
    if [[ ${JOB_NAME} == *openmp* ]]; then
        export CC=/usr/local/opt/llvm/bin/clang
        export CXX=/usr/local/opt/llvm/bin/clang++
    fi
244
245
fi

246
###############################################################################
247
248
# Set up the location for the local object store outside of the build and
# source tree, which can be shared by multiple builds.
249
250
# It defaults to a MantidExternalData directory within the HOME directory.
# It can be overridden by setting the MANTID_DATA_STORE environment variable.
251
252
###############################################################################
if [ -z "$MANTID_DATA_STORE" ]; then
253
    export MANTID_DATA_STORE=$HOME/MantidExternalData
254
255
fi

256
257
258
###############################################################################
# Packaging options
###############################################################################
259
if [[ ${DO_BUILD_PKG} == true ]]; then
260
    PACKAGINGVARS="-DPACKAGE_DOCS=ON -DDOCS_DOTDIAGRAMS=ON -DDOCS_SCREENSHOTS=ON -DDOCS_MATH_EXT=sphinx.ext.imgmath -DDOCS_PLOTDIRECTIVE=ON"
261
262
263
264
265
266
267
268
269
270
271
272
273
274
    # Use different suffix if parameter is not defined
    if [[ -n "${PACKAGE_SUFFIX}" ]]; then
        echo "Using PACKAGE_SUFFIX=${PACKAGE_SUFFIX} from job parameter"
    elif [[ ${JOB_NAME} == release* ]]; then  # starts with "release"
        PACKAGE_SUFFIX=""
    elif [[ ${JOB_NAME} == ornl-stable ]]; then
        PACKAGE_SUFFIX=""
    elif [[ ${JOB_NAME} == ornl-qa ]]; then
        PACKAGE_SUFFIX="qa"
    elif [[ ${JOB_NAME} == master* ]]; then
        PACKAGE_SUFFIX="nightly"
    else
        PACKAGE_SUFFIX="unstable"
    fi
275
    if [[ "${ON_MACOS}" == true ]]; then
276
        PACKAGINGVARS="${PACKAGINGVARS} -DCPACK_PACKAGE_SUFFIX=${PACKAGE_SUFFIX}"
277
    else
278
279
280
        if [[ ${JOB_NAME} == release* ]] && [[ -z "${PACKAGE_SUFFIX}" ]]; then
            # Traditional install path for release build
            PACKAGINGVARS="${PACKAGINGVARS} -DCMAKE_INSTALL_PREFIX=/opt/Mantid -DCPACK_PACKAGE_SUFFIX="
281
        elif [[ ${JOB_NAME} == ornl-stable ]] && [[ -z "${PACKAGE_SUFFIX}" ]]; then
Martyn Gigg's avatar
Martyn Gigg committed
282
            # Traditional install path for release build
283
284
285
286
287
            PACKAGINGVARS="${PACKAGINGVARS} -DCMAKE_INSTALL_PREFIX=/opt/Mantid -DCPACK_PACKAGE_SUFFIX="
        else
            # everything else uses lower-case values
            PACKAGINGVARS="${PACKAGINGVARS} -DCMAKE_INSTALL_PREFIX=/opt/mantid${PACKAGE_SUFFIX} -DCPACK_PACKAGE_SUFFIX=${PACKAGE_SUFFIX}"
        fi
288

289
        if [[ ${ON_RHEL7} == true ]]; then
290
            if [[ -z "${RELEASE_NUMBER}" ]]; then
291
292
293
294
                RELEASE_NUMBER="1"
            fi
            PACKAGINGVARS="${PACKAGINGVARS} -DCPACK_RPM_PACKAGE_RELEASE=${RELEASE_NUMBER}"
        fi
295
296
297
298

        if [[ -n "${GITHUB_AUTHORIZATION_TOKEN}" ]]; then
            PACKAGINGVARS="${PACKAGINGVARS} -DGITHUB_AUTHORIZATION_TOKEN=${GITHUB_AUTHORIZATION_TOKEN}"
        fi
299
    fi
300
fi
301

302
303
304
305
306
307
###############################################################################
# Figure out if were doing a sanitizer build and setup any steps we need
###############################################################################

if [[ ${JOB_NAME} == *address* ]]; then
    SANITIZER_FLAGS="-DUSE_SANITIZER=Address"
308

309
elif [[ ${JOB_NAME} == *memory* ]]; then
310
    SANITIZER_FLAGS="-DUSE_SANITIZER=memory"
311

312
elif [[ ${JOB_NAME} == *thread* ]]; then
313
    SANITIZER_FLAGS="-DUSE_SANITIZER=thread"
314

315
elif [[ ${JOB_NAME} == *undefined* ]]; then
316
317
318
    SANITIZER_FLAGS="-DUSE_SANITIZER=undefined"
fi

319
if [[ -n "${SANITIZER_FLAGS}" ]]; then
320
321
322
    # Force build to RelWithDebInfo
    BUILD_CONFIG="RelWithDebInfo"
fi
323

324
325
326
###############################################################################
# Generator
###############################################################################
327
if [ "$(command -v ninja)" ]; then
328
329
330
    CMAKE_GENERATOR="-G Ninja"
    elif [ "$(command -v ninja-build)" ]; then
    CMAKE_GENERATOR="-G Ninja"
331
fi
332

333
if [ -e $BUILD_DIR/CMakeCache.txt ]; then
334
    CMAKE_GENERATOR=""
335
336
fi

337
###############################################################################
338
# Work in the build directory
339
###############################################################################
340
cd $BUILD_DIR
341
342

###############################################################################
343
344
# Clean up any artifacts from last build so that if it fails
# they don't get archived again
345
###############################################################################
346
rm -f -- *.dmg *.rpm *.deb *.tar.gz *.tar.xz
347
348
349
350

###############################################################################
# CMake configuration
###############################################################################
351
$SCL_ENABLE "${CMAKE_EXE} ${CMAKE_GENERATOR} -DCMAKE_BUILD_TYPE=${BUILD_CONFIG} -DENABLE_PRECOMMIT=OFF -DENABLE_CPACK=ON -DMANTID_DATA_STORE=${MANTID_DATA_STORE} -DDOCS_HTML=ON -DENABLE_CONDA=ON -DCOLORED_COMPILER_OUTPUT=OFF ${DIST_FLAGS} ${PACKAGINGVARS} ${CLANGTIDYVAR} ${SANITIZER_FLAGS} .."
352

353
354
355
356
###############################################################################
# Coverity build should exit early
###############################################################################
if [[ ${JOB_NAME} == *coverity_build_and_submit* ]]; then
357
358
359
360
361
362
363
364
365
366
367
368
    ${COVERITY_DIR}/cov-build --dir cov-int ${CMAKE_EXE} --build . -- -j ${BUILD_THREADS:?}
    tar czvf mantid.tgz cov-int
    status=$(curl --form token=$COVERITY_TOKEN --form email=mantidproject@gmail.com \
        --form file=@mantid.tgz --form version=$GIT_COMMIT \
    https://scan.coverity.com/builds?project=mantidproject%2Fmantid)
    status=$(echo ${status} | sed -e 's/^ *//' -e 's/ *$//')
    if [[ -z $status ]] || [[ ${status} == "Build successfully submitted." ]]; then
        exit 0
    else
        echo "$status"
        exit 1
    fi
369
370
fi

371
372
373
###############################################################################
# Build step
###############################################################################
374
if [[ ${DO_BUILD_CODE} == true ]]; then
375
376
    ${CMAKE_EXE} --build . -- -j${BUILD_THREADS:?}
    ${CMAKE_EXE} --build . --target AllTests -- -j${BUILD_THREADS:?}
377
fi
378

379
###############################################################################
380
# Static analysis builds or stop here
381
382
###############################################################################
if [[ $USE_CLANG ]] && [[ ${JOB_NAME} == *clang_tidy* ]]; then
383
    exit 0
384
385
fi

386
###############################################################################
387
# Run the unit tests
388
###############################################################################
389
390
391
# Activate core dumps. They are deactivated by the registered EXIT function
# at the top of this script
if [[ ${USE_CORE_DUMPS} == true ]]; then
392
    ulimit -c unlimited
393
fi
394
395
396
# Prevent race conditions when creating the user config directory
userconfig_dir=$HOME/.mantid
rm -fr $userconfig_dir
397
398
399
400
# Remove old application saved state & crash reports on mac.
# If we don't do this then when a previous test has crashed macOS
# will pop up a dialog box and wait for clicking okay. We are heavy
# handed but builders tend not to be used for anything else.
401
if [[ ${ON_MACOS} == true ]] ; then
402
403
    rm -fr "$HOME/Library/Saved Application State/org.python.python"
    rm -f $HOME/Library/Application\ Support/CrashReporter/*
404
fi
405
# Remove GUI qsettings files
406
407
rm -f ~/.config/mantidproject/mantidworkbench.ini

408
mkdir -p $userconfig_dir
409
# use a fixed number of openmp threads to avoid overloading the system
410
411
userprops_file=$userconfig_dir/Mantid.user.properties
echo MultiThreaded.MaxCores=2 > $userprops_file
412

413
if [[ ${DO_UNITTESTS} == true ]]; then
414
    run_with_xvfb $CTEST_EXE --no-compress-output -T Test -j${BUILD_THREADS:?} --schedule-random --output-on-failure
415
    terminate_xvfb_sessions
416
fi
417

418
###############################################################################
419
# User Documentation
420
###############################################################################
421
if [[ ${DO_DOCTESTS_USER} == true ]]; then
422
423
424
425
426
427
428
429
430
    # use default configuration
    rm -f $userprops_file
    # Remove doctrees directory so it forces a full reparse. It seems that
    # without this newly added doctests are not executed
    if [ -d $BUILD_DIR/docs/doctrees ]; then
        rm -fr $BUILD_DIR/docs/doctrees/*
    fi
    # Build HTML to verify that no referencing errors have crept in.
    run_with_xvfb ${CMAKE_EXE} --build . --target docs-html
431
    terminate_xvfb_sessions
432
    run_with_xvfb ${CMAKE_EXE} --build . --target docs-doctest
433
434
fi

435
436
437
438
439
440
441
442
443
444
445
###############################################################################
# Developer Documentation
###############################################################################
# Uncomment this when the dev-docs are ready to build without warnings
# if [[ ${DO_BUILD_DEVDOCS} == true ]]; then
#   if [ -d $BUILD_DIR/dev-docs/doctree ]; then
#     rm -fr $BUILD_DIR/dev-docs/doctree/*
#   fi
#   ${CMAKE_EXE} --build . --target dev-docs-html
# fi

446
###############################################################################
447
448
# Create the install kit if required. This includes building the Qt help
# documentation
449
###############################################################################
450
if [[ ${DO_BUILD_PKG} == true ]]; then
451
452
    run_with_xvfb ${CMAKE_EXE} --build . --target docs-qthelp
    ${CPACK_EXE}
453

454
    # Source tarball on clean build (arbitrarily choose Ubuntu)
455
456
457
    # Also, parcel up the documentation into a tar file that is easier to move around
    # and labelled by the commit id it was built with. This assumes the Jenkins git plugin
    # has set the GIT_COMMIT environment variable
458
    if [[ ${CLEANBUILD} == true && ${ON_UBUNTU} == true ]]; then
459
460
461
462
463
        run_with_xvfb ${CMAKE_EXE} --build . --target docs-html
        tar -cjf mantiddocs-g${GIT_COMMIT:0:7}.tar.bz2 --exclude='*.buildinfo' --exclude="MantidProject.q*" docs/html
        # The ..._PREFIX argument avoids opt/Mantid directories at the top of the tree
        ${CPACK_EXE} --config CPackSourceConfig.cmake -D CPACK_PACKAGING_INSTALL_PREFIX=
    fi
464
fi
465
466

###############################################################################
467
468
# Run the system tests if required. Run from a package to have at least one
# Linux checks it install okay
469
###############################################################################
470
if [[ ${DO_SYSTEMTESTS} == true ]]; then
471
472
473
474
475
    if [[ ${PRBUILD} == true ]]; then
        EXTRA_ARGS="--exclude-in-pull-requests" $SCRIPT_DIR/systemtests
    else
        $SCRIPT_DIR/systemtests
    fi
476
fi