Commit ed973c09 authored by Morales Hernandez, Mario's avatar Morales Hernandez, Mario
Browse files

Merge branch 'master' into 'master' 

parallel input implemented

See merge request hydro/triton!1
parents c41d82f7 eb5a124c

README.md

+34 −0
Original line number Diff line number Diff line
@@ -34,3 +34,37 @@ Highest TRITON computational efficiency can be achieved by using GPU implementat 




Morales-Hernández, M., Sharif, M.B., Kalyanapu, A., Ghafoor, S.K., Dullo, T.T., Gangrade, S., Kao, S.C., Norman, M.R. and Evans, K.J., 2021. TRITON: A Multi-GPU Open Source 2D Hydrodynamic Flood Model. Environmental Modelling & Software, p.105034 [https://doi.org/10.1016/j.envsoft.2021.105034](https://doi.org/10.1016/j.envsoft.2021.105034)





**Instructions to run a case with parallel input**



1.  Open the script "scriptSplitASCII" and configure the following parameters (in caps):



*  TRITON_DIR: triton directory

*  NFILES: The number of files to split the ASCII file(s) into. It should match the number of ranks in the TRITON simulation

*  INPUT_DEM: DEM input file (in ASCII format)

*  IS_MANN: flag for the mann file (YES or NO)

*  INPUT_MANN: in case IS_MANN=YES, this parameter contains the mann input file (in ASCII format)

*  IS_RMAP: flag for the rmap file (YES or NO)

*  INPUT_RMAP: in case IS_RMAP=YES, this parameter contains the rmap input file (in ASCII format)

*  OUTPUT_FORMAT: ASC or BIN, depending on the desired output format for the sequence of files

*  ASCII2BIN_FOLDER: in case OUTPUT_FORMAT=BIN, it points to the ascii2bin directory (not the ascii2bin_dem)

*  ASCII2BIN_RMAP_FOLDER: in case OUTPUT_FORMAT=BIN and IS_RMAP=YES, it points to the ascii2bin_rmap directory







2.  Run the script.  





3.  Configure the cfg file as follows for the following parameters:



*  dem_filename: include the path of the split files plus the basename (without underscores or numbers). Example: "input/dem/bin/par/case03"

*  Add header_filename and point it to the exact name of the header that the script has generated. Example: "input/dem/bin/par/case03.header" 

*  n_infile: include the path of the split files plus the basename (without underscores or numbers). Example: "input/mann/bin/par/case03"

*  runoff_map: include the path of the split files plus the basename (without underscores or numbers). Example: ""input/runoff/bin/par/case03_runoff"

*  input_format: BIN

*  Add input_option: PAR



4.  Run TRITON with the same MPI ranks as stated in NFILES.

input/cfg/case03_parallel.cfg

0 → 100755
+86 −0
Original line number Diff line number Diff line 
#---------------------------------------------------------------------------

# Triton config file - Case 03

#---------------------------------------------------------------------------

# Input Data

#---------------------------------------------------------------------------

# Topography 

dem_filename="input/dem/bin/par/case03"

header_filename="input/dem/bin/par/case03.header"



# Manning input files

n_infile="input/mann/bin/par/case03"

#const_mann=0.035



# Hydrograph and flow locations information

hydrograph_filename="input/strmflow/case03.hyg"

num_sources=2

src_loc_file="input/strmflow/case03.txt"

hydrograph_row_size=41



# Runoff related information

runoff_row_size=41

num_runoffs=156

runoff_filename="input/runoff/case03_runoff.hyg"

runoff_map="input/runoff/bin/par/case03_runoff"

#---------------------------------------------------------------------------

# Output Control

#---------------------------------------------------------------------------

# print_option=h to output just the h, huv to output all h,u and v

print_option=huv



# Print interval in seconds of simulation time

print_interval=1800



# time_series_flag=1 to activate, 0 to deactivate

observation_loc_file="input/obs/case5.obs"

time_series_flag=0



#---------------------------------------------------------------------------

# Simulation Control

#---------------------------------------------------------------------------

# Start and duration of simulation time in seconds

sim_start_time=0

sim_duration=86400



# If checkpoint_id is 0 that means a clean start

checkpoint_id=0



# time_increment_fixed=0 is for variable dt, 1 for constant dt

time_increment_fixed=0

time_step=0.01



# Initial input files

#h_infile=""

#qx_infile=""

#qy_infile=""



#---------------------------------------------------------------------------

# Input Output File Settings

#---------------------------------------------------------------------------

# File Format=BIN or ASC

# File Output Option=PAR or SEQ



input_format=BIN

input_option=PAR

output_format=BIN

outfile_pattern="%s/%s/%s_%02d_%02d"

output_option=PAR

#---------------------------------------------------------------------------

# External boundaries

#---------------------------------------------------------------------------

num_extbc=2

extbc_file="input/extbc/case03.extbc"



#---------------------------------------------------------------------------

# Other variables

#---------------------------------------------------------------------------

it_count=0

# Courant Number

courant=0.5



hextra=0.001

gpu_direct_flag=0

domain_decomposition=static

factor_interval_domain_decomposition=2

open_boundaries=0

scriptSplitASCII

0 → 100755
+165 −0
Original line number Diff line number Diff line 
#!/bin/bash



function pwait() {

    while [ $(jobs -p | wc -l) -ge $1 ]; do

        sleep 1

    done

}

cpus=$( ls -d /sys/devices/system/cpu/cpu[[:digit:]]* | wc -w )



#takes an ASCII DEM and/or MANN/RMAP files and split them into a header (always ASCII) and a number of ASCII or binary files



#variables to be defined by the user (in caps)

TRITON_DIR="/home/user/fork_tritonmpi/tritonmpi" #TRITON folder

NFILES=8 #number of files to be split into. It should match the number of subdomains (nranks) in the TRITON simulation

INPUT_DEM="${TRITON_DIR}/input/dem/asc/case03.dem" #dem file

IS_MANN="YES" #YES or NO

INPUT_MANN="${TRITON_DIR}/input/mann/asc/case03.mann" #mann file (needed in case IS_MANN=YES)

IS_RMAP="YES" #YES or NO

INPUT_RMAP="${TRITON_DIR}/input/runoff/case03_runoff.rmap" #rmap file (needed in case IS_RMAP=YES)

OUTPUT_FORMAT="BIN" #ASCII or BIN

ASCII2BIN_FOLDER="${TRITON_DIR}/tools/ascii2bin/" #ascii2bin folder (needed in case BIN=YES)

ASCII2BIN_RMAP_FOLDER="${TRITON_DIR}/tools/ascii2bin_rmap/" #ascii2bin_rmap folder (needed in case BIN=YES and IS_RMAP=YES)

#-----------------------------------------------------------------------------------------------------------------------#



#get the input file until find the character

input_dir="${INPUT_DEM%/*}"

#get the number of rows

nrows=$(sed -ne '2{p;q;}' $INPUT_DEM | cut -c 6-)



#delete until the last . occurrence

casename_dem="${INPUT_DEM%.*}"

casename_mann="${INPUT_MANN%.*}"

casename_rmap="${INPUT_RMAP%.*}"



#create the header file in the same dir as the input dem file

head -n 6 $INPUT_DEM > "$casename_dem.header"





#the partition way should match the way it's done in TRITON

nlines=$((nrows / NFILES ))



rem=$((nrows % NFILES));



sum=0

for i in $(seq 0 $((NFILES-2)))

do

	line_numbers[$i]=$((sum+nlines))

	sum=$((sum+nlines))

	if [ $i -lt $rem ] && [ $rem -gt 0 ]

	then

		((line_numbers[$i]++))

		((sum++))

	fi

done



start_index=1                 # The offset to calculate lines

idx=0                   # The index used in the name of generated files: file1, file2 ...



for i in "${line_numbers[@]}"

do

   # Extract the lines using the head and tail commands

	#pipe the file without the header (6 lines that's why the 7) and split it

	digit=$(printf '%02d\n' $idx)

	tail -n+7 $INPUT_DEM | tail -n +$start_index | head -n $(( i-start_index+1 )) > "$casename_dem"_"$digit".dem & 

	pwait $cpus

	if [[ $IS_MANN == "YES" ]]

	then

		tail -n +$start_index $INPUT_MANN | head -n $(( i-start_index+1 )) > "$casename_mann"_"$digit".mann &

		pwait $cpus

	fi

	if [[ $IS_RMAP == "YES" ]]

	then

		tail -n +$start_index $INPUT_RMAP | head -n $(( i-start_index+1 )) > "$casename_rmap"_"$digit".rmap &

		pwait $cpus

	fi

	 echo "Split ASCII file" $idx



    (( idx++ ))

    start_index=$(( i+1 ))

done



# Extract the last given lines - last lines in the file

idx=$((NFILES-1))

start_index=$((${line_numbers[$NFILES-2]}+1))



# Extract the last given lines - last lines in the file

digit=$(printf '%02d\n' $idx)

tail -n+7 $INPUT_DEM | tail -n +$start_index > "$casename_dem"_"$digit".dem

if [[ $IS_MANN == "YES" ]]

then

	tail -n +$start_index $INPUT_MANN > "$casename_mann"_"$digit".mann

fi

if [[ $IS_RMAP == "YES" ]]

then

	tail -n +$start_index $INPUT_RMAP > "$casename_rmap"_"$digit".rmap

fi

echo "Split ASCII file" $idx



echo "ASCII files generated"





#convert to binary

if [[ $OUTPUT_FORMAT == "BIN" ]]

then

	echo "Converting to binary..."



	#DEM files

	input_folder="${INPUT_DEM%/*}/par/"

	mkdir -p $input_folder && cd $input_folder && rm *.*

	mv "$casename_dem"_*.dem .  

	cd ../../

	output_folder=$(echo "$PWD/bin/par/")

	mkdir -p $output_folder

	cd $ASCII2BIN_FOLDER

	for f in "$input_folder"*.*

	do

		filename=$(basename $f)

		./ascii2bin "$input_folder$filename" "$output_folder$filename" 1 &

		pwait $cpus

	done

	cd $output_folder

	mv "$casename_dem".header .



	#MANN FILES

	if [[ $IS_MANN == "YES" ]]

	then

		input_folder="${INPUT_MANN%/*}/par/"

		mkdir -p $input_folder && cd $input_folder && rm *.*

		mv "$casename_mann"_*.mann .  

		cd ../../

		output_folder=$(echo "$PWD/bin/par/")

		mkdir -p $output_folder

		cd $ASCII2BIN_FOLDER

		for f in "$input_folder"*.*

		do

			filename=$(basename $f)

			./ascii2bin "$input_folder$filename" "$output_folder$filename" 1 &

			pwait $cpus

		done

	fi



	if [[ $IS_RMAP == "YES" ]]

	then

		#RMAP FILES

		input_folder="${INPUT_RMAP%/*}/asc/par/"

		mkdir -p $input_folder && cd $input_folder && rm *.*

		mv "$casename_rmap"_*.rmap .  

		cd ../../

		output_folder=$(echo "$PWD/bin/par/")

		mkdir -p $output_folder

		cd $ASCII2BIN_RMAP_FOLDER

		for f in "$input_folder"*.*

		do

			filename=$(basename $f)

			./ascii2bin_rmap "$input_folder$filename" "$output_folder$filename" 1 &

			pwait $cpus

		done



	fi



	cd $TRITON_DIR

	echo "Succeed! Files converted"

fi

src/config_utils.h

+15 −3
Original line number Diff line number Diff line
@@ -55,9 +55,11 @@ namespace ConfigUtils 
		print_option,	/**< Use to determine output types. h to output just the h (depth), huv to output all h (depth),u and v (velocities). */

		max_value_print_option,	/**< Use to determine maximum value of each cells output types. h to output just the h (depth). */

		input_format,	/**< Expected input file format. BIN for binary file or ASC for ascii file. */

		input_option,	/**< Strategy to use for input files. PAR for parallel input or SEQ for sequential inp. PAR reads each MPI partitions subdomain in separate files and SEQ reads the whole domain from one file. Applied to all raster formats. By default, SEQ is considered.*/

		output_format,	/**< Expected output file format. BIN for binary file or ASC for ascii file. */

		output_option,	/**< Strategy to use for outputting into files. PAR for parallel outputs or SEQ for sequential outputs. PAR saves each MPI partitions subdomain in separate files and SEQ saves the whole domain into one file. */

		dem_filename,	/**< Directory of the DEM file to use. */

		header_filename,	/**< Directory of the header file to use (in case of parallel reading). */

		src_loc_file,	/**< Directory of the file that contains the information of all flow locations. */

		runoff_map,	/**< Directory of the Runoff map to use. */

		observation_loc_file,	/**< Directory of the file that contains the information of all cells to observe and generate time series output. */
@@ -421,13 +423,14 @@ namespace ConfigUtils 
		arguments<T> arglist;



		arglist.open_boundaries=0; //by default

		arglist.input_option = "SEQ"; //by default



		arglist.outfile_pattern = argsd("outfile_pattern", argmap, "");

		arglist.hydrograph_filename = args("hydrograph_filename", argmap);

		arglist.runoff_filename = args("runoff_filename", argmap);

		arglist.dem_filename = args("dem_filename", argmap);

		arglist.dem_filename = args("dem_filename", argmap); //in the case of PAR input, the dem file should include only the name (without extension and _)

		arglist.src_loc_file = argsd("src_loc_file", argmap, "");

		arglist.runoff_map = argsd("runoff_map", argmap, "");

		arglist.runoff_map = argsd("runoff_map", argmap, ""); //in the case of PAR input, the rmap file should include only the name (without extension and _)

		arglist.observation_loc_file = argsd("observation_loc_file", argmap, "");

		arglist.extbc_file = argsd("extbc_file", argmap, "");

		arglist.extbc_dir = argsd("extbc_dir", argmap, "");
@@ -449,13 +452,14 @@ namespace ConfigUtils 
		arglist.print_option = StringUtils::tolower(args("print_option", argmap));

		arglist.max_value_print_option = argsd("max_value_print_option", argmap, "");

		arglist.input_format = args("input_format", argmap);

		arglist.input_option = args("input_option", argmap);

		arglist.output_format = args("output_format", argmap);

		arglist.output_option = args("output_option", argmap);



		arglist.h_infile = argsd("h_infile", argmap, "");

		arglist.qx_infile = argsd("qx_infile", argmap, "");

		arglist.qy_infile = argsd("qy_infile", argmap, "");

		arglist.n_infile = argsd("n_infile", argmap, "");

		arglist.n_infile = argsd("n_infile", argmap, "");  //in the case of PAR input, the n_infile file should include only the name (without extension and _)



		arglist.domain_decomposition = args("domain_decomposition", argmap);

		arglist.factor_interval_domain_decomposition = atoi((args("factor_interval_domain_decomposition", argmap)).c_str());
@@ -490,6 +494,14 @@ namespace ConfigUtils 
			arglist.extbc_fname = StringUtils::split((args("extbc_fname", extbc_map)), ',');

		}

		

		if(strcmp(arglist.input_option.c_str(), "")==0){ //no input_option provided (for legacy version)

			arglist.input_option = "SEQ"; //by default

		}



		if(strcmp(arglist.input_option.c_str(), "PAR")==0){

			arglist.header_filename = args("header_filename", argmap);

		}



		return arglist;

	}

src/constants.h

+1 −1
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ typedef double value_t; /**< Data type to represent floating-point number. It 
#define H 0    /**< Water depth array position in vector. */

#define QX 1    /**< Flux X array position in vector. */

#define QY 2    /**< Flux Y array position in vector. */

#define N 3    /**< Manning array position in vector. */

#define NMAN 3    /**< Manning array position in vector. */

#define DEM 4    /**< DEM array position in vector. */

#define MAXH 5    /**< Max values of water depth array position in vector. */

#define RHSH0 6    /**< Partial water depth 1 array position in vector. */