Commit 95c3673e authored by William F Godoy's avatar William F Godoy

Dump per Level Rank size

Added DataStructures dependency for OrderedDict
Add Summit location
parent 5c0652b1
...@@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" ...@@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a" GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a"
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
[compat] [compat]
julia = "1" julia = "1"
......
...@@ -11,7 +11,10 @@ function main() ...@@ -11,7 +11,10 @@ function main()
app::String = "AmrexCastro" app::String = "AmrexCastro"
# location where several run outputs are located they must have a common prefix # location where several run outputs are located they must have a common prefix
outputPrefix::String = "/home/wgodoy/workspace/Castro/Exec/hydro_tests/Sedov/case" # Local
# outputPrefix::String = "/home/wgodoy/workspace/Castro/Exec/hydro_tests/Sedov/case"
# Summit location
outputPrefix::String = "/gpfs/alpine/proj-shared/csc310/wgodoy/amrex/Castro/Exec/hydro_tests/Sedov/case"
# file name inside outputs directories # file name inside outputs directories
logFile::String = "run.log" logFile::String = "run.log"
......
...@@ -13,9 +13,11 @@ Pkg.add("Plots") ...@@ -13,9 +13,11 @@ Pkg.add("Plots")
Pkg.add("PackageCompiler") Pkg.add("PackageCompiler")
Pkg.add("Flux") Pkg.add("Flux")
Pkg.add("CSV") Pkg.add("CSV")
Pkg.add("DataStructures")
using PackageCompiler using PackageCompiler
create_sysimage([:Glob, :Plots, :GLM, :DataFrames, :Flux, :CSV], sysimage_path = "jexio_deps.so") create_sysimage([:Glob, :Plots, :GLM, :DataFrames, :Flux, :CSV, :DataStructures],
sysimage_path = "jexio_deps.so")
exit() exit()
...@@ -5,6 +5,7 @@ import DataFrames ...@@ -5,6 +5,7 @@ import DataFrames
import GLM import GLM
import Plots import Plots
import CSV import CSV
import DataStructures
struct AmrexCastro <: AbstractAmrex struct AmrexCastro <: AbstractAmrex
app::String app::String
...@@ -179,25 +180,24 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame ...@@ -179,25 +180,24 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
# output cumulative data size, per timestep, level, rank # output cumulative data size, per timestep, level, rank
Yd_Timesteps::Array{Int64,1} = [] Yd_Timesteps::Array{Int64,1} = []
Yd_Levels::Dict{ String,Array{Int64,1} } = Dict() Yd_Levels::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
Yd_Ranks::Dict{ String,Array{Int64,1} } = Dict() Yd_Ranks::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
nlevels::Int64 = _find_max_nlevels(outputDirs) nlevels::Int64 = _find_max_nlevels(outputDirs)
nranks::Int64 = _find_max_nranks(outputDirs) nranks::Int64 = _find_max_nranks(outputDirs)
for level in (1:nlevels) for level in (1:nlevels)
keyLevel::String = string("Level_",level-1) levelKey::String = string("Level_",level-1)
push!(Yd_Levels, keyLevel => []) push!(Yd_Levels, levelKey => [])
println("Yd_Levels", Yd_Levels, " ", level) println("Yd_Levels", Yd_Levels, " ", level)
for rank in (1:nranks) for rank in (1:nranks)
rankID = lpad(rank-1,5,"0") rankKey::String = string(levelKey,"/Cell_D_", lpad(rank-1,5,"0") )
keyLevelRank::String = string("Level_",level-1, "/Cell_D_",rankID ) push!(Yd_Ranks, rankKey => [])
push!(Yd_Ranks, keyLevelRank => [])
end end
end end
println("Yd_Ranks: ", Yd_Ranks) println("Yd_Ranks: ", Yd_Ranks)
caseIDs::Array{String,1} = [] caseIDs::Array{String,1} = []
...@@ -228,7 +228,6 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame ...@@ -228,7 +228,6 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
# Yd timesteps # Yd timesteps
_push_datasize!(plotFileDir, Yd_Timesteps) _push_datasize!(plotFileDir, Yd_Timesteps)
levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_")) levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
currentLevels = helper_get_relative_path.(levelDirs) currentLevels = helper_get_relative_path.(levelDirs)
...@@ -236,24 +235,36 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame ...@@ -236,24 +235,36 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
# Levels # Levels
for (levelKey,levelSizeValues) in Yd_Levels for (levelKey,levelSizeValues) in Yd_Levels
levelDir = string(plotFileDir,"/",levelKey)
foundArray = findall( x -> x == levelKey, currentLevels) foundArray = findall( x -> x == levelKey, currentLevels)
if size(foundArray,1) == 1 if size(foundArray,1) == 1
levelDir = string(plotFileDir,"/",levelKey)
_push_datasize!(levelDir, Yd_Levels[levelKey]) _push_datasize!(levelDir, Yd_Levels[levelKey])
# add ranks
rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
rankFiles = helper_get_relative_path.(rankFullFiles)
for rankFile in rankFiles
end
else else
push!(Yd_Levels[levelKey],0) push!(Yd_Levels[levelKey],0)
end end
end # Level Rank data
# add ranks
rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
rankFiles = helper_get_relative_path.(rankFullFiles)
for rankID in 1:nranks
rankKey::String = string(levelKey,"/Cell_D_", lpad(rankID-1,5,"0") )
# check if file exist
checkFileName = string(levelDir,"/Cell_D_", lpad(rankID-1,5,"0") )
if isfile( checkFileName )
rankFileSize = filesize(checkFileName)
push!(Yd_Ranks[rankKey], rankFileSize)
else
push!(Yd_Ranks[rankKey],0)
end
end
end
# caseID # caseID
push!(caseIDs, caseID ) push!(caseIDs, caseID )
...@@ -272,7 +283,11 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame ...@@ -272,7 +283,11 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
for (key,Yd_Level) in Yd_Levels for (key,Yd_Level) in Yd_Levels
df[!,key] = Yd_Level df[!,key] = Yd_Level
end end
for (key,Yd_Rank) in Yd_Ranks
df[!,key] = Yd_Rank
end
df[!,"caseID"] = caseIDs df[!,"caseID"] = caseIDs
df[!,"caseDir"] = caseDirs df[!,"caseDir"] = caseDirs
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment