From 5c0652b1611ee6d76c50e1873a52b19043c594e4 Mon Sep 17 00:00:00 2001 From: William F Godoy Date: Wed, 23 Sep 2020 18:45:29 -0400 Subject: [PATCH] Adjusting to max level found --- src/extractor/AmrexCastro.jl | 165 ++++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 40 deletions(-) diff --git a/src/extractor/AmrexCastro.jl b/src/extractor/AmrexCastro.jl index 256eccd..9acbe2d 100644 --- a/src/extractor/AmrexCastro.jl +++ b/src/extractor/AmrexCastro.jl @@ -107,37 +107,112 @@ end function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrames.DataFrame) - # using Flux ML for 1st order fitting + function _push_datasize!(directory::String, data::Array{Int64,1}) + if size(data,1) == 0 + push!(data, helper_get_directory_size(directory) ) + else + push!(data, last(data) + helper_get_directory_size(directory) ) + end + end + + function _find_max_nlevels(outputDirs::Array{String})::Int64 + + max_nlevels::Int64 = 0 + + for outputDir in outputDirs + inputFile::String = _get_input_file(extractor, outputDir) + parameters = _input_parser(extractor, inputFile) + rootPlotName = get(parameters, "amr.plot_file", "") + + # find all directories with rootPlotName and get its size + # root of _plt directories + plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName)) + + for plotFileDir in plotFileDirs + levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_")) + nlevels = size(levelDirs,1) + + if max_nlevels < nlevels + max_nlevels = nlevels + end + end + end + + return max_nlevels + end + + + function _find_max_nranks(outputDirs::Array{String})::Int64 + + max_nranks::Int64 = 0 + for outputDir in outputDirs + + inputFile::String = _get_input_file(extractor, outputDir) + parameters = _input_parser(extractor, inputFile) + rootPlotName = get(parameters, "amr.plot_file", "") + + # find all directories with rootPlotName and get its size + # root of _plt directories + plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName)) + + for plotFileDir in plotFileDirs + + levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_")) + + for levelDir in levelDirs + rankDirs = helper_get_prefix_directories(string(levelDir,"/Cell_D_")) + nranks = size(rankDirs,1) + + if max_nranks < nranks + max_nranks = nranks + end + end + end + end + return max_nranks + end + + # output directories + outputDirs = helper_get_prefix_directories(extractor.outputPrefix) # input with total_cells * output_number Xd::Array{Int64,1} = [] - # output directory size - Yd::Array{Float64,1} = [] - # output Level0 directory size - Yd_Level0::Array{Float64,1} = [] - Yd_Level1::Array{Float64,1} = [] - Yd_Level2::Array{Float64,1} = [] + # output cumulative data size, per timestep, level, rank + Yd_Timesteps::Array{Int64,1} = [] + + Yd_Levels::Dict{ String,Array{Int64,1} } = Dict() + Yd_Ranks::Dict{ String,Array{Int64,1} } = Dict() + + nlevels::Int64 = _find_max_nlevels(outputDirs) + nranks::Int64 = _find_max_nranks(outputDirs) + + for level in (1:nlevels) + keyLevel::String = string("Level_",level-1) + push!(Yd_Levels, keyLevel => []) + println("Yd_Levels", Yd_Levels, " ", level) + for rank in (1:nranks) + rankID = lpad(rank-1,5,"0") + keyLevelRank::String = string("Level_",level-1, "/Cell_D_",rankID ) + push!(Yd_Ranks, keyLevelRank => []) + end + end + + + + println("Yd_Ranks: ", Yd_Ranks) caseIDs::Array{String,1} = [] caseDirs::Array{String,1} = [] - - outputDirs = helper_get_prefix_directories(extractor.outputPrefix) for (index,outputDir) in enumerate(outputDirs) inputFile::String = _get_input_file(extractor, outputDir) parameters = _input_parser(extractor, inputFile) rootPlotName = get(parameters, "amr.plot_file", "") - # rootCheckpointName = get(parameters, "amr.plot_file", "") # find all directories with rootPlotName and get its size # root of _plt directories plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName)) - plotsSize::Int64 = 0 - plotsSizeLevel0::Int64 = 0 - plotsSizeLevel1::Int64 = 0 - plotsSizeLevel2::Int64 = 0 - counter::Int32 = 1 # extract ncells @@ -150,27 +225,36 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame # Xd push!( Xd, counter*ncells) - # Yd - sizeDir::Int64 = helper_get_directory_size(plotFileDir) - plotsSize += sizeDir - push!( Yd, plotsSize) - - # Yd_Level0 - sizeDirLevel0::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_0")) - plotsSizeLevel0 += sizeDirLevel0 - push!( Yd_Level0, plotsSizeLevel0) + # Yd timesteps + _push_datasize!(plotFileDir, Yd_Timesteps) - # Yd_Level1 - sizeDirLevel1::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_1")) - plotsSizeLevel1 += sizeDirLevel1 - push!( Yd_Level1, plotsSizeLevel1) - # Yd_Level2 - sizeDirLevel2::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_2")) - plotsSizeLevel2 += sizeDirLevel2 - push!( Yd_Level2, plotsSizeLevel2) + levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_")) + currentLevels = helper_get_relative_path.(levelDirs) + println("Current levels: ", currentLevels) + # Levels + for (levelKey,levelSizeValues) in Yd_Levels + + foundArray = findall( x -> x == levelKey, currentLevels) + + if size(foundArray,1) == 1 + levelDir = string(plotFileDir,"/",levelKey) + _push_datasize!(levelDir, Yd_Levels[levelKey]) + + # add ranks + rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_")) + rankFiles = helper_get_relative_path.(rankFullFiles) + for rankFile in rankFiles + + end + else + push!(Yd_Levels[levelKey],0) + end + + end + # caseID push!(caseIDs, caseID ) @@ -182,16 +266,17 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame end - data = [(Xd,Yd)] - df::DataFrames.DataFrame = DataFrames.DataFrame( plots_cells = Xd, - plots_size = Yd, - plots_size_L0 = Yd_Level0, - plots_size_L1 = Yd_Level1, - plots_size_L2 = Yd_Level2, - caseID = caseIDs, - caseDir = caseDirs) + plots_size = Yd_Timesteps) + + for (key,Yd_Level) in Yd_Levels + df[!,key] = Yd_Level + end + + df[!,"caseID"] = caseIDs + df[!,"caseDir"] = caseDirs + println(df) CSV.write("plot_size.csv", df, header=true) -- GitLab