Adjusting to max level found

src/extractor/AmrexCastro.jl

@@ -107,37 +107,112 @@ end
 
 function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrames.DataFrame) 
 
-  # using Flux ML for 1st order fitting
+  function _push_datasize!(directory::String, data::Array{Int64,1})
+    if size(data,1) == 0
+      push!(data, helper_get_directory_size(directory) )
+    else
+      push!(data, last(data) + helper_get_directory_size(directory) )
+    end
+  end
+  
+  function _find_max_nlevels(outputDirs::Array{String})::Int64
+    
+    max_nlevels::Int64 = 0
+    
+    for outputDir in outputDirs
+      inputFile::String = _get_input_file(extractor, outputDir)
+      parameters = _input_parser(extractor, inputFile)
+      rootPlotName = get(parameters, "amr.plot_file", "")
+
+      # find all directories with rootPlotName and get its size
+      # root of _plt directories 
+      plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
+      
+      for plotFileDir in plotFileDirs
+        levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
+        nlevels = size(levelDirs,1)  
+      
+        if max_nlevels < nlevels
+          max_nlevels = nlevels
+        end
+      end
+    end
+    
+    return max_nlevels
+  end
+  
+  
+  function _find_max_nranks(outputDirs::Array{String})::Int64
+    
+    max_nranks::Int64 = 0
+    for outputDir in outputDirs
+      
+      inputFile::String = _get_input_file(extractor, outputDir)
+      parameters = _input_parser(extractor, inputFile)
+      rootPlotName = get(parameters, "amr.plot_file", "")
+
+      # find all directories with rootPlotName and get its size
+      # root of _plt directories 
+      plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
+      
+      for plotFileDir in plotFileDirs
+      
+        levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
+      
+        for levelDir in levelDirs
+          rankDirs = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
+          nranks = size(rankDirs,1)
+        
+          if max_nranks < nranks
+            max_nranks = nranks
+          end
+        end
+      end
+    end  
+    return max_nranks
+  end
+    
+  # output directories
+  outputDirs = helper_get_prefix_directories(extractor.outputPrefix)
   # input with total_cells * output_number
   Xd::Array{Int64,1} = []
-  # output directory size
-  Yd::Array{Float64,1} = []
-  # output Level0 directory size
-  Yd_Level0::Array{Float64,1} = []
-  Yd_Level1::Array{Float64,1} = []
-  Yd_Level2::Array{Float64,1} = []
+  # output cumulative data size, per timestep, level, rank
+  Yd_Timesteps::Array{Int64,1} = []
+
+  Yd_Levels::Dict{ String,Array{Int64,1} } = Dict()
+  Yd_Ranks::Dict{ String,Array{Int64,1} } = Dict()
+  
+  nlevels::Int64 =  _find_max_nlevels(outputDirs)
+  nranks::Int64 = _find_max_nranks(outputDirs)
+  
+  for level in (1:nlevels)   
+    keyLevel::String = string("Level_",level-1)
+    push!(Yd_Levels, keyLevel => [])
+    println("Yd_Levels", Yd_Levels, " ", level)
+    for rank in (1:nranks)
+      rankID = lpad(rank-1,5,"0")
+      keyLevelRank::String = string("Level_",level-1, "/Cell_D_",rankID )
+      push!(Yd_Ranks, keyLevelRank => [])
+    end
+  end
+  
+     
+      
+  println("Yd_Ranks: ", Yd_Ranks)
     
   caseIDs::Array{String,1} = []
   caseDirs::Array{String,1} = []
-
-  outputDirs = helper_get_prefix_directories(extractor.outputPrefix)
       
   for (index,outputDir) in enumerate(outputDirs)
     
     inputFile::String = _get_input_file(extractor, outputDir)
     parameters = _input_parser(extractor, inputFile)
     rootPlotName = get(parameters, "amr.plot_file", "")
-    # rootCheckpointName = get(parameters, "amr.plot_file", "")
 
     # find all directories with rootPlotName and get its size
     # root of _plt directories 
     plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
 
-    plotsSize::Int64 = 0
-    plotsSizeLevel0::Int64 = 0
-    plotsSizeLevel1::Int64 = 0
-    plotsSizeLevel2::Int64 = 0
-    
     counter::Int32 = 1
     
     # extract ncells
@@ -150,27 +225,36 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
       # Xd
       push!( Xd, counter*ncells)
       
-      # Yd
-      sizeDir::Int64 = helper_get_directory_size(plotFileDir)
-      plotsSize += sizeDir
-      push!( Yd, plotsSize)
-      
-      # Yd_Level0
-      sizeDirLevel0::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_0"))
-      plotsSizeLevel0 += sizeDirLevel0
-      push!( Yd_Level0, plotsSizeLevel0)
+      # Yd timesteps
+      _push_datasize!(plotFileDir, Yd_Timesteps)
       
-      # Yd_Level1
-      sizeDirLevel1::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_1"))
-      plotsSizeLevel1 += sizeDirLevel1
-      push!( Yd_Level1, plotsSizeLevel1)
       
-      # Yd_Level2
-      sizeDirLevel2::Int64 = helper_get_directory_size(string(plotFileDir, "/Level_2"))
-      plotsSizeLevel2 += sizeDirLevel2
-      push!( Yd_Level2, plotsSizeLevel2)
+      levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
+      currentLevels = helper_get_relative_path.(levelDirs)
       
+      println("Current levels: ", currentLevels)
       
+      # Levels
+      for (levelKey,levelSizeValues) in Yd_Levels
+        
+        foundArray = findall( x -> x == levelKey, currentLevels)
+        
+        if size(foundArray,1) == 1
+          levelDir = string(plotFileDir,"/",levelKey)
+           _push_datasize!(levelDir, Yd_Levels[levelKey])
+           
+           # add ranks
+           rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
+           rankFiles = helper_get_relative_path.(rankFullFiles)
+           for rankFile in rankFiles
+              
+           end
+        else
+           push!(Yd_Levels[levelKey],0)
+        end
+        
+      end 
+        
       # caseID
       push!(caseIDs, caseID )
       
@@ -182,16 +266,17 @@ function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrame
     
   end
 
-  data = [(Xd,Yd)]
-  
   df::DataFrames.DataFrame = 
       DataFrames.DataFrame( plots_cells = Xd, 
-                            plots_size = Yd, 
-                            plots_size_L0 = Yd_Level0,
-                            plots_size_L1 = Yd_Level1,
-                            plots_size_L2 = Yd_Level2,
-                            caseID = caseIDs, 
-                            caseDir = caseDirs)
+                            plots_size = Yd_Timesteps)
+                            
+  for (key,Yd_Level) in Yd_Levels
+    df[!,key] = Yd_Level
+  end                          
+  
+  df[!,"caseID"] = caseIDs
+  df[!,"caseDir"] = caseDirs
+
   println(df)
   
   CSV.write("plot_size.csv",  df, header=true)