Dump per Level Rank size

Added DataStructures dependency for OrderedDict
Add Summit location

Project.toml

@@ -9,6 +9,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 GLM = "38e38edf-8417-5370-95a0-9cbb8c7f171a"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 
 [compat]
 julia = "1"

examples/ExampleAmrexCastro.jl

@@ -11,7 +11,10 @@ function main()
     app::String = "AmrexCastro"
 
     # location where several run outputs are located they must have a common prefix 
-    outputPrefix::String = "/home/wgodoy/workspace/Castro/Exec/hydro_tests/Sedov/case"
+    # Local
+    # outputPrefix::String = "/home/wgodoy/workspace/Castro/Exec/hydro_tests/Sedov/case"
+    # Summit location
+    outputPrefix::String = "/gpfs/alpine/proj-shared/csc310/wgodoy/amrex/Castro/Exec/hydro_tests/Sedov/case"
 
     # file name inside outputs directories
     logFile::String = "run.log"

scripts/requirements.jl

@@ -13,9 +13,11 @@ Pkg.add("Plots")
 Pkg.add("PackageCompiler")
 Pkg.add("Flux")
 Pkg.add("CSV")
+Pkg.add("DataStructures")
 
 
 using PackageCompiler
-create_sysimage([:Glob, :Plots, :GLM, :DataFrames, :Flux, :CSV], sysimage_path = "jexio_deps.so")
+create_sysimage([:Glob, :Plots, :GLM, :DataFrames, :Flux, :CSV, :DataStructures], 
+                 sysimage_path = "jexio_deps.so")
 
 exit()

src/extractor/AmrexCastro.jl

@@ -5,6 +5,7 @@ import DataFrames
 import GLM
 import Plots
 import CSV
+import DataStructures
 
 struct AmrexCastro <: AbstractAmrex
     app::String
@@ -179,24 +180,23 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
   # output cumulative data size, per timestep, level, rank
   Yd_Timesteps::Array{Int64,1} = []
 
-  Yd_Levels::Dict{ String,Array{Int64,1} } = Dict()
-  Yd_Ranks::Dict{ String,Array{Int64,1} } = Dict()
+  Yd_Levels::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
+  Yd_Ranks::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
   
   nlevels::Int64 =  _find_max_nlevels(outputDirs)
   nranks::Int64 = _find_max_nranks(outputDirs)
   
   for level in (1:nlevels)   
-    keyLevel::String = string("Level_",level-1)
-    push!(Yd_Levels, keyLevel => [])
+    levelKey::String = string("Level_",level-1)
+    push!(Yd_Levels, levelKey => [])
     println("Yd_Levels", Yd_Levels, " ", level)
+    
     for rank in (1:nranks)
-      rankID = lpad(rank-1,5,"0")
-      keyLevelRank::String = string("Level_",level-1, "/Cell_D_",rankID )
-      push!(Yd_Ranks, keyLevelRank => [])
-    end
+      rankKey::String = string(levelKey,"/Cell_D_", lpad(rank-1,5,"0") )
+      push!(Yd_Ranks, rankKey => [])
     end
     
-     
+  end
   
   println("Yd_Ranks: ", Yd_Ranks)
     
@@ -228,7 +228,6 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
       # Yd timesteps
       _push_datasize!(plotFileDir, Yd_Timesteps)
       
-      
       levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
       currentLevels = helper_get_relative_path.(levelDirs)
       
@@ -237,20 +236,32 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
       # Levels
       for (levelKey,levelSizeValues) in Yd_Levels
 
+        levelDir = string(plotFileDir,"/",levelKey)
         foundArray = findall( x -> x == levelKey, currentLevels)
         
         if size(foundArray,1) == 1
-          levelDir = string(plotFileDir,"/",levelKey)
            _push_datasize!(levelDir, Yd_Levels[levelKey])
+        else
+           push!(Yd_Levels[levelKey],0)
+        end
         
+        # Level Rank data
         # add ranks 
         rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
         rankFiles = helper_get_relative_path.(rankFullFiles)
-           for rankFile in rankFiles
            
-           end
+        for rankID in 1:nranks
+            
+            rankKey::String = string(levelKey,"/Cell_D_", lpad(rankID-1,5,"0") )
+            # check if file exist
+            checkFileName =  string(levelDir,"/Cell_D_", lpad(rankID-1,5,"0") )
+            
+            if isfile( checkFileName )
+              rankFileSize = filesize(checkFileName)
+              push!(Yd_Ranks[rankKey], rankFileSize)
             else
-           push!(Yd_Levels[levelKey],0)
+              push!(Yd_Ranks[rankKey],0)  
+            end
         end
         
       end
@@ -274,6 +285,10 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
     df[!,key] = Yd_Level
   end
   
+  for (key,Yd_Rank) in Yd_Ranks
+    df[!,key] = Yd_Rank
+  end
+  
   df[!,"caseID"] = caseIDs
   df[!,"caseDir"] = caseDirs