Commit ebefa82e authored by William F Godoy's avatar William F Godoy

Working on checkpoint-restart

parent 549bc3c2
......@@ -47,8 +47,9 @@ function run_linear_models(extractor::AmrexCastro)
for key in keys(extractor.outputs)
if (key == "plots_size")
#_run_linear_model_plots_size(extractor, X)
_run_linear_models_plots_size_step(extractor, X)
elseif(key == "checkpoint_size")
#_run_linear_models_checkpoint_size_step(extractor, X)
end
end
end
......@@ -108,71 +109,6 @@ end
function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrames.DataFrame)
function _push_datasize!(directory::String, data::Array{Int64,1})
if size(data,1) == 0
push!(data, helper_get_directory_size(directory) )
else
push!(data, last(data) + helper_get_directory_size(directory) )
end
end
function _find_max_nlevels(outputDirs::Array{String})::Int64
max_nlevels::Int64 = 0
for outputDir in outputDirs
inputFile::String = _get_input_file(extractor, outputDir)
parameters = _input_parser(extractor, inputFile)
rootPlotName = get(parameters, "amr.plot_file", "")
# find all directories with rootPlotName and get its size
# root of _plt directories
plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
for plotFileDir in plotFileDirs
levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
nlevels = size(levelDirs,1)
if max_nlevels < nlevels
max_nlevels = nlevels
end
end
end
return max_nlevels
end
function _find_max_nranks(outputDirs::Array{String})::Int64
max_nranks::Int64 = 0
for outputDir in outputDirs
inputFile::String = _get_input_file(extractor, outputDir)
parameters = _input_parser(extractor, inputFile)
rootPlotName = get(parameters, "amr.plot_file", "")
# find all directories with rootPlotName and get its size
# root of _plt directories
plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
for plotFileDir in plotFileDirs
levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
for levelDir in levelDirs
rankDirs = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
nranks = size(rankDirs,1)
if max_nranks < nranks
max_nranks = nranks
end
end
end
end
return max_nranks
end
# output directories
outputDirs = helper_get_prefix_directories(extractor.outputPrefix)
# input with total_cells * output_number
......@@ -183,8 +119,8 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
Yd_Levels::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
Yd_Ranks::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
nlevels::Int64 = _find_max_nlevels(outputDirs)
nranks::Int64 = _find_max_nranks(outputDirs)
nlevels::Int64 = _find_max_nlevels(extractor, outputDirs, "plot")
nranks::Int64 = _find_max_nranks(extractor, outputDirs, "plot")
for level in (1:nlevels)
levelKey::String = string("Level_",level-1)
......@@ -202,13 +138,14 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
caseIDs::Array{String,1} = []
caseDirs::Array{String,1} = []
parameterKey = "amr.plot_file"
for (index,outputDir) in enumerate(outputDirs)
inputFile::String = _get_input_file(extractor, outputDir)
parameters = _input_parser(extractor, inputFile)
rootPlotName = get(parameters, "amr.plot_file", "")
rootPlotName = get(parameters, parameterKey, "")
# find all directories with rootPlotName and get its size
# root of _plt directories
plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
......@@ -260,24 +197,22 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
rankFiles = helper_get_relative_path.(rankFullFiles)
for rankID in 1:nranks
rankKey::String = string(levelKey,"/Cell_D_", lpad(rankID-1,5,"0") )
# check if file exist
checkFileName = string(levelDir,"/Cell_D_", lpad(rankID-1,5,"0") )
if isfile( checkFileName )
rankFileSize = filesize(checkFileName)
# bug push!(Yd_Ranks[rankKey], rankFileSize)
if plotIndex == 1
push!(Yd_Ranks[rankKey], rankFileSize)
# push!( Yd_Levels[levelKey], helper_get_directory_size(levelDir) )
push!(Yd_Ranks[rankKey], rankFileSize)
else
push!( Yd_Ranks[rankKey], last(Yd_Ranks[rankKey]) + rankFileSize )
end
else
push!(Yd_Ranks[rankKey],0)
push!(Yd_Ranks[rankKey],0)
end
end
......@@ -316,3 +251,87 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
#readline()
end
# private functions
"""
Finds the max number of levels N scanning the subdirectories of outputDirs (Level_0, Level_1, ..., Level_N-1)
:param: extractor input
:param: outputDirs output directories to scan for max level
:param: file_type either "plot" or "check"
"""
function _find_max_nlevels(extractor::AmrexCastro, outputDirs::Array{String}, file_type::String)::Int64
max_nlevels::Int64 = 0
for outputDir in outputDirs
inputFile::String = _get_input_file(extractor, outputDir)
parameters = _input_parser(extractor, inputFile)
if file_type == "plot"
rootPlotName = get(parameters, "amr.plot_file", "")
elseif file_type == "check"
rootPlotName = get(parameters, "amr.check_file", "")
end
# find all directories with rootPlotName and get its size
# root of _plt directories
plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
for plotFileDir in plotFileDirs
levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
nlevels = size(levelDirs,1)
if max_nlevels < nlevels
max_nlevels = nlevels
end
end
end
return max_nlevels
end
function _find_max_nranks(extractor::AmrexCastro, outputDirs::Array{String}, file_type::String)::Int64
max_nranks::Int64 = 0
for outputDir in outputDirs
inputFile::String = _get_input_file(extractor, outputDir)
parameters = _input_parser(extractor, inputFile)
if file_type == "plot"
rootPlotName = get(parameters, "amr.plot_file", "")
elseif file_type == "check"
rootPlotName = get(parameters, "amr.check_file", "")
end
# find all directories with rootPlotName and get its size
# root of _plt directories
plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
for plotFileDir in plotFileDirs
levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
for levelDir in levelDirs
rankDirs = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
nranks = size(rankDirs,1)
if max_nranks < nranks
max_nranks = nranks
end
end
end
end
return max_nranks
end
function _push_datasize!(directory::String, data::Array{Int64,1})
if size(data,1) == 0
push!(data, helper_get_directory_size(directory) )
else
push!(data, last(data) + helper_get_directory_size(directory) )
end
end
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment