Adds script to build ananconda with mpi5py and h5py on Titan.

#!/bin/bash

# Deploy Anaconda with key packages built using dependencies provided by OLCF

# environment modules.

#=============================================================================

# Overview

# ========

# There are several key considerations to running your own python interpereter

# (like anaconda) in a virtualenv on Titan:

#

#  * All environment module changes must be made while no python venvs are

#    active.

#  * OLCF provided python environment modules (python_*) CANNOT be used with

#    your own self-installed python iterperator. Make sure none are in your

#    environment. If you need one we provide, it must be installed in your venv

#    (see mpi4py below).

#  * Pre-compiled binaries should be avoided for packages that use dependencies

#    provided by OLCF environment modules

#  * The cray compiler wrapper needs to be used (instead of the raw system GCC

#    compiler) when building packages with exotic link dependencies with

#    OLCF-provided environment modules.

#  * Compiled code that will be executed in-part on the batch nodes must be

#    built targetting them, even if the code may also be run on the compute

#    nodes.

# This script is written to explain what it is doing at each step in light of

# the above considerations. Feel free to edit any part of it to suite your

# needs. Once configured, this script should produce and actrivate a consistent

# python runtime environment when sourced.

# Basic setup

# ===========

# Point me to where you want anaconda intalled.

# I recommend "/ccs/proj/<PROJID>/opt/anaconda/titan-gnu" which labels the host

# and programming environment, is readable on the compute nodes, and is not

# purged.

ANACONDA_PATH="${ANACONDA_PATH-/ccs/proj/csc103/opt/anaconda/titan-gnu}"

VENV_NAME="myenv"

# Set the environment modules

# ===========================

# ALL ENVIRONMENT MODULE OPERATIONS MUST BE DONE OUTSIDE A PYTHON VIRTUALENV!

# Use the GNU Programming environment because most python packages will be

# off-the-shelf configured to use gcc. Using another compiler will make this

# much harder.

module unload PrgEnv-pgi

module load PrgEnv-gnu

# Load all packages that provide dependencies. Add anything else that is missing!

module load cray-hdf5-parallel

# Deploy Anaconda

# ===============

# On M.P.Belhorn's preferred shell (zsh), the anaconda 'activate' script doesn't

# work without being in the anaconda binary directory. It assumes we have an

# up-to-date zsh. This is to recall the CWD from which this script was sourced.

_LAST_CWD="$PWD"

# Install anaconda as needed

if [ ! -d "${ANACONDA_PATH}" ]; then 

  # Use a particular anaconda installer version.

  ANACONDA_INSTALLER="Anaconda2-4.2.0-Linux-x86_64.sh"

  if [ ! -f "./${ANACONDA_INSTALLER}" ]; then 

    wget "https://repo.continuum.io/archive/${ANACONDA_INSTALLER}"

    chmod a+x "${ANACONDA_INSTALLER}"

  fi 

  ./${ANACONDA_INSTALLER} -p "${ANACONDA_PATH}"

  rm "./${ANACONDA_INSTALLER}"

fi 

# Add Anconda to the PATH

# =======================

# We want to use ONLY the anaconda distro we 

# We cannot block Python from finding things in your $HOME/.local space, but be

# careful that items there may have been installed/built against other

# interperetors and in the context of other environments. Try to avoid using

# packages installed there unless they are simple, v2.6-compatible packages

# written purely in python.

export PATH="${ANACONDA_PATH}/bin:$PATH"

# Wipe the PYTHONPATH so it is not contaminated with packages installed for use

# with OLCF-provided python iterpereters.

export PYTHONPATH=''

# Activate the venv

# =================

# Strictly speaking, this isn't necessary since we just went through the work of

# installing our own interperator but it is good practice and prevents some

# clashes with OLCF provided python packages/interperators.

cd "$ANACONDA_PATH/bin"

# If this is the first time this script has been called,

# then create the venv...

if [ ! -d "../envs/myenv" ]; then

  # Configure compiler wrapper to target batch node so compiled code will run on

  # both batch and compute nodes. Switch back to craype-interlagos when building

  # code that will only be run on the compute nodes for optimal performance.

  module unload craype-interlagos

  module load craype-istanbul

  # Create the venv

  conda create -n $VENV_NAME python=2.7

  # Activate it

  source activate $VENV_NAME

  # Manually install core packages without relying on precompiled binaries from

  # conda that would use, for instance, openmpi instead of Cray MPICH.

  pip install --upgrade pip

  CC=cc pip install -v --no-binary :all: mpi4py

  # H5py needs some configure options to use MPI. Worst case scenerio, some

  # pacakges will need to be built without pip.

  CC=cc HDF5_MPI="ON" HDF5_DIR="${CRAY_HDF5_DIR}" pip install -v --no-binary :all: h5py

  source deactivate $VENV_NAME

  # Reconfigure the compiler wrapper for the compute nodes. Remember, all

  # environment module changes must happen *without* an active python venv.

  module unload craype-istanbul

  module load craype-interlagos

fi

# Activate the venv.

source activate $VENV_NAME

# End where we began.

cd "$_LAST_CWD