revert cooldown utility and remove scaling application.

# Build applications.

add_subdirectory(applications)

################################################################################

# Cooldown scrpts

file(COPY ${CMAKE_SOURCE_DIR}/tools/cooldown DESTINATION ${CMAKE_BINARY_DIR}/)

################################################################################

# Scaling scrpts

file(GLOB MY_FILES ${CMAKE_SOURCE_DIR}/applications/scaling/*)

file(COPY ${MY_FILES} DESTINATION ${CMAKE_BINARY_DIR}/scaling/)

###############################################################################

# Generate a .cpp file with DCA++'s CMake options and their values.

configure_file("${PROJECT_SOURCE_DIR}/src/config/cmake_options.cpp.in"

  "${PROJECT_BINARY_DIR}/src/config/cmake_options.cpp" @ONLY)

# Generates directories, input files and batch scripts for a cooldown, i.e. a series of dca/analysis

# runs with gradually decreased temperature.

# By default, this script and the provided template input files (input_sp.json.in and

# input_sp.json.in) are configured for DCA(+) calculations of the 2D single-band Hubbard model with

# on-site Coulomb interaction U and fixed density d.

#

# Usage: 1. Configure the EDIT block.

#        2. Execute the script

#

# Author: Giovanni Balduzzi (gbalduzz@itp.phys.ethz.ch)

import os

################################################################################

##################################### EDIT #####################################

################################################################################

# Numbers of compute nodes

node_counts = [1, 2, 4, 8, 16, 32, 64, 128]

# Measurements

measurements_per_node = True # Set to False for strong scaling.

measurements = 5000          # Something like 10 to 20M for strong scaling.

# System: options: "titan" "summit"

system= "titan"

# Code version: options: "old" "new"

version= "new"

# Submit: if true submit the script after generation.

submit = False

################################################################################

################################## UNTIL HERE ##################################

################################################################################

################################################################################

############################ System Specific settings ##########################

################################################################################

########## Try to optimize parameters depending on system and version #########

walkers = 0

accumulators = 0

shared = False

smt = 1

threads = 7 if system == "summit" else 8

if system == "summit" and version == "new" :

    walkers = accumulators = 7

    shared = True

elif system == "summit" and version == "old" :

    walkers = 7

    accumulators = 7

    smt = 2

elif system == "titan" and version == "new" :

    walkers = 5

    accumulators = 1

elif system == "titan" and version == "old" :

    walkers = 3

    accumulators = 11

else : raise "Invalid options."

################################################################################

# Inverse temperature

beta = 50

# DCA cluster (see cluster_definitions.txt)

size_x = 6

size_y = 6

################################################################################

################################################################################

################################################################################

def boolToStr(boolean):

    return "true"  if boolean else "false"

# Prepares the input file for the given temperature.

def prepare_input_file(filename, outname, nodes):

    file = open(filename, 'r')

    text = file.read()

    file.close()

    text = text.replace('BETA_CURRENT_TEMP', str(beta))

    text = text.replace('NODES', str(nodes))

    text = text.replace('SIZE_X', str(size_x))

    text = text.replace('SIZE_Y', str(size_y))

    tot_meas = measurements * nodes if measurements_per_node else measurements

    text = text.replace('MEASUREMENTS', str(tot_meas))

    text = text.replace('ACCUMULATORS', str(accumulators))

    text = text.replace('WALKERS', str(walkers))

    text = text.replace('SHARED', boolToStr(shared))

    text = text.replace('THREADS', str(threads))

    file = open(outname, 'w')

    file.write(text)

    file.close()

################################################################################

batch_tmpl = system + '_scaling.job.in'

print('Generating directories and input files:')

run_type = "weak" if measurements_per_node else "strong"

parent_dir = "run_dca_" + system + "_" + version + "_" + run_type

for n_ind, n in enumerate(node_counts):

    print('N = ' + str(n))

    # Create directory.

    dir_str = parent_dir + '/nodes_' + str(n)

    os.system('mkdir -p ' + dir_str)

    # Generate the input file.

    input = dir_str + '/input.json'

    prepare_input_file('input.json.in', input, n)

    # Add job.

    # Generate the dca batch script.

    batch_name_dca = parent_dir + "/dca_nodes_" + str(n) +  ".job"

    print('\nGenerating the dca batch script: ' + batch_name_dca)

    file = open(batch_tmpl, 'r')

    text_dca = file.read()

    file.close()

    executable =  '../main_dca_' + version

    text_dca = text_dca.replace('EXEC', executable)

    text_dca = text_dca.replace('INPUT', './nodes_' + str(n) + "/input.json")

    text_dca = text_dca.replace('OUTPUT', './nodes_' + str(n) + '/output.txt')

    text_dca = text_dca.replace('SMT', str(smt))

    text_dca = text_dca.replace('NODES', str(n))

    processes_per_node = 6 if system == "summit" else 1

    text_dca = text_dca.replace('PROCESSES', str(processes_per_node * n))

    file = open(batch_name_dca, 'w')

    file.write(text_dca)

    file.close()

    os.chmod(batch_name_dca, 0o755)

    if submit :

        submission_exec = "bsub" if system == "summit" else "qsub"

        os.system(submission_exec + " " + batch_name_dca)

{

    "output": {

     "directory" : "./nodes_NODES/",

     "output-format": "HDF5",

     "filename-dca": "dca.hdf5",

     "filename-profiling": "profiling.json",

     "directory-config-read" : "../beta_BETA_CURRENT_TEMP/configuration",

     "dump-lattice-self-energy": false,

     "dump-cluster-Greens-functions": true,

     "dump-Gamma-lattice": false,

     "dump-chi-0-lattice": false

    },

    "physics": {

        "beta": 50,

        "density": 1,

        "chemical-potential": 0.,

        "adjust-chemical-potential": false

    },

    "single-band-Hubbard-model": {

        "t": 1.,

        "U": 4

    },

    "DCA": {

        "initial-self-energy" : "../beta_BETA_CURRENT_TEMP/dca_sp.hdf5",

        "iterations": 1,

        "accuracy": 0.,

        "self-energy-mixing-factor": 1.,

        "interacting-orbitals": [0],

        "do-finite-size-QMC": false,

        "coarse-graining": {

            "k-mesh-recursion": 3,

            "periods": 2,

            "quadrature-rule": 1,

            "threads": THREADS,

            "tail-frequencies": 0

        },

        "DCA+": {

            "do-DCA+": false,

            "deconvolution-iterations": 16,

            "deconvolution-tolerance": 1.e-2

        }

    },

    "domains": {

        "real-space-grids": {

            "cluster": [[SIZE_X, 0],

                        [0, SIZE_Y]],

            "sp-host": [[20, 0],

                        [0, 20]],

            "tp-host": [[8, 0],

                        [0, 8]]

        },

        "imaginary-time": {

            "sp-time-intervals": 1024

        },

        "imaginary-frequency": {

            "sp-fermionic-frequencies": 1024,

            "four-point-fermionic-frequencies": 64

        }

    },

    "four-point": {

        "type": "PARTICLE_PARTICLE_UP_DOWN",

        "compute-all-transfers" : true,

        "frequency-transfer" : 0

    },

    "Monte-Carlo-integration": {

        "seed": 0,

        "warm-up-sweeps": 10,

        "sweeps-per-measurement": 1,

        "measurements": MEASUREMENTS,

	"error-computation-type" : "JACK_KNIFE",

        "threaded-solver": {

            "walkers": WALKERS,

            "accumulators": ACCUMULATORS,

            "shared-walk-and-accumulation-thread": SHARED

        }

    },

    "CT-AUX": {

        "expansion-parameter-K": 1.,

        "max-submatrix-size": 128,

        "neglect-Bennett-updates": false,

        "additional-time-measurements": false

    }

}

#!/bin/bash

# Begin LSF Directives

#BSUB -P CPH102

#BSUB -W 2:00

#BSUB -nnodes NODES

#BSUB -alloc_flags smtSMT   ## set simultaneous multithreading level (no. of hardware threads per physical core)

#BSUB -J scaling_NODES

#BSUB -o scaling_summit_NODES.%J

#BSUB -e scaling_summit_NODES.%J

# Change into scratch filesystem

cd $LS_SUBCWD

# Load required modules

source $MODULESHOME/init/bash

module reset

module load gcc/6.4.0

module load cuda

module load hdf5

module load fftw

module load cmake

module load magma

module load netlib-lapack

module load essl

date

jsrun -n PROCESSES -a 1 -g 1 -c 7 -b rs EXEC INPUT > OUTPUT

# TODO: compute l2 norm on the G4 error.

date

#!/bin/bash

#

#PBS -A cph102

#PBS -N scaling_NODES

#PBS -o scaling_titan_NODES.$PBS_JOBID

#PBS -e scaling_titan_NODES.$PBS_JOBID

#PBS -l nodes=NODES

#PBS -l walltime=02:00:00

#PBS -l gres=atlas1%atlas2

export OMP_NUM_THREADS=1

module reset

module unload PrgEnv-pgi

module load PrgEnv-gnu

module load cray-hdf5

module load cray-fftw

module load cudatoolkit

module load magma/2.4.0

module load cmake3

cd $PBS_O_WORKDIR

date

#TODO: check it and add number of threads.

aprun -n PROCESSES EXEC INPUT > OUTPUT

date