AmrexCastro.jl 8.46 KB
Newer Older
William F Godoy's avatar
William F Godoy committed
1

2
include("../helper/helperSystem.jl")
3

William F Godoy's avatar
William F Godoy committed
4 5
import DataFrames
import GLM
6 7
import Plots
import CSV
William F Godoy's avatar
William F Godoy committed
8
import DataStructures
William F Godoy's avatar
William F Godoy committed
9 10

struct AmrexCastro <: AbstractAmrex
William F Godoy's avatar
William F Godoy committed
11 12
    app::String
    outputPrefix::String
William F Godoy's avatar
William F Godoy committed
13 14
    inputs::Array{String}
    outputs::Dict{String,Array{String}}
William F Godoy's avatar
William F Godoy committed
15
    runlogFile::String
William F Godoy's avatar
William F Godoy committed
16

William F Godoy's avatar
William F Godoy committed
17
    AmrexCastro(outputPrefix::String, runlogFile::String) = new(
William F Godoy's avatar
William F Godoy committed
18 19 20 21 22 23 24 25 26 27 28
        "AmrexCastro",
        outputPrefix,
        [
            "max_step",
            "amr.check_int",
            "amr.plot_int",
            "amr.n_cell",
            "amr.max_level",
            "castro.cfl",
            "castro.max_grid_size",
        ],
William F Godoy's avatar
William F Godoy committed
29
        Dict{String,Array{String}}(
William F Godoy's avatar
William F Godoy committed
30
            "plots_size" => ["amr.nplot_files", "amr.ncells"],
31 32 33 34 35 36 37
            "checks_size" => [
                "amr.check_int",
                "amr.ncheck_files",
                "amr.ncells",
                "amr.max_level",
                "amr.regrid_int",
            ],
William F Godoy's avatar
William F Godoy committed
38 39
        ),
        runlogFile,
William F Godoy's avatar
William F Godoy committed
40
    )
William F Godoy's avatar
William F Godoy committed
41 42
end

William F Godoy's avatar
William F Godoy committed
43 44
# runs the linear models for each entry in outputs
function run_linear_models(extractor::AmrexCastro)
William F Godoy's avatar
William F Godoy committed
45

William F Godoy's avatar
William F Godoy committed
46
    X::DataFrames.DataFrame = _get_linear_model_X(extractor)
William F Godoy's avatar
William F Godoy committed
47

William F Godoy's avatar
William F Godoy committed
48
    for key in keys(extractor.outputs)
William F Godoy's avatar
William F Godoy committed
49
        if (key == "plots_size")
50 51
            #_run_linear_model_plots_size(extractor, X)
            _run_linear_models_plots_size_step(extractor, X)
William F Godoy's avatar
William F Godoy committed
52
        end
William F Godoy's avatar
William F Godoy committed
53 54
    end
end
55

William F Godoy's avatar
William F Godoy committed
56

William F Godoy's avatar
William F Godoy committed
57
function _get_input_file(extractor::AmrexCastro, outputDir::String)::String
William F Godoy's avatar
William F Godoy committed
58

William F Godoy's avatar
William F Godoy committed
59 60
    # hardcoded input...find out if this is always the case with Castro   
    inputFiles::Array{String} = helper_get_prefix_files("input", outputDir)
William F Godoy's avatar
William F Godoy committed
61

William F Godoy's avatar
William F Godoy committed
62
    if isempty(inputFiles)
William F Godoy's avatar
William F Godoy committed
63 64 65 66
        throw(NoSuchFieldException(
            "jexio AmrexCastro: Can't find Castro input files in directories:",
            outputDirs,
        ))
William F Godoy's avatar
William F Godoy committed
67
        # use length, instead of size as it returns tuples
William F Godoy's avatar
William F Godoy committed
68 69 70 71 72
    elseif length(inputFiles) != 1
        print("Number of files: ", size(inputFiles))
        throw(DimensionMismatch("jexio AmrexCastro: found more than one input file"))
    else
    end
73

William F Godoy's avatar
William F Godoy committed
74
    # print(inputFiles)
William F Godoy's avatar
William F Godoy committed
75 76
    return inputFiles[1]
end
William F Godoy's avatar
William F Godoy committed
77 78 79 80 81 82



function _get_linear_model_X(extractor::AmrexCastro)::DataFrames.DataFrame
    # get directories runs output location
    outputDirs = helper_get_prefix_directories(extractor.outputPrefix)
William F Godoy's avatar
William F Godoy committed
83
    println(outputDirs)
William F Godoy's avatar
William F Godoy committed
84 85 86

    X::DataFrames.DataFrame = DataFrames.DataFrame()
    isXInit::Bool = false
William F Godoy's avatar
William F Godoy committed
87

William F Godoy's avatar
William F Godoy committed
88 89
    for outputDir in outputDirs
        inputFile::String = _get_input_file(extractor, outputDir)
William F Godoy's avatar
William F Godoy committed
90 91
        # get relevant input parameters
        parameters = _input_parser(extractor, inputFile)
William F Godoy's avatar
William F Godoy committed
92 93 94
        independentVariables = _get_independent_variables(extractor, parameters)
        # get outputDir name
        independentVariables["caseID"] = helper_get_relative_path(outputDir)
William F Godoy's avatar
William F Godoy committed
95

William F Godoy's avatar
William F Godoy committed
96
        if isXInit == false
William F Godoy's avatar
William F Godoy committed
97
            X = DataFrames.DataFrame(independentVariables)
William F Godoy's avatar
William F Godoy committed
98 99
            isXInit = true
        else
William F Godoy's avatar
William F Godoy committed
100
            DataFrames.push!(X, independentVariables)
William F Godoy's avatar
William F Godoy committed
101 102 103 104
        end
    end

    println(X)
William F Godoy's avatar
William F Godoy committed
105
    return X
106 107 108
end


109 110
function _run_linear_models_plots_size_step(extractor::AmrexCastro, X::DataFrames.DataFrame) 

William F Godoy's avatar
William F Godoy committed
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
  function _push_datasize!(directory::String, data::Array{Int64,1})
    if size(data,1) == 0
      push!(data, helper_get_directory_size(directory) )
    else
      push!(data, last(data) + helper_get_directory_size(directory) )
    end
  end
  
  function _find_max_nlevels(outputDirs::Array{String})::Int64
    
    max_nlevels::Int64 = 0
    
    for outputDir in outputDirs
      inputFile::String = _get_input_file(extractor, outputDir)
      parameters = _input_parser(extractor, inputFile)
      rootPlotName = get(parameters, "amr.plot_file", "")

      # find all directories with rootPlotName and get its size
      # root of _plt directories 
      plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
      
      for plotFileDir in plotFileDirs
        levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
        nlevels = size(levelDirs,1)  
      
        if max_nlevels < nlevels
          max_nlevels = nlevels
        end
      end
    end
    
    return max_nlevels
  end
  
  
  function _find_max_nranks(outputDirs::Array{String})::Int64
    
    max_nranks::Int64 = 0
    for outputDir in outputDirs
      
      inputFile::String = _get_input_file(extractor, outputDir)
      parameters = _input_parser(extractor, inputFile)
      rootPlotName = get(parameters, "amr.plot_file", "")

      # find all directories with rootPlotName and get its size
      # root of _plt directories 
      plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))
      
      for plotFileDir in plotFileDirs
      
        levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
      
        for levelDir in levelDirs
          rankDirs = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
          nranks = size(rankDirs,1)
        
          if max_nranks < nranks
            max_nranks = nranks
          end
        end
      end
    end  
    return max_nranks
  end
    
  # output directories
  outputDirs = helper_get_prefix_directories(extractor.outputPrefix)
178 179
  # input with total_cells * output_number
  Xd::Array{Int64,1} = []
William F Godoy's avatar
William F Godoy committed
180 181 182
  # output cumulative data size, per timestep, level, rank
  Yd_Timesteps::Array{Int64,1} = []

William F Godoy's avatar
William F Godoy committed
183 184
  Yd_Levels::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
  Yd_Ranks::DataStructures.SortedDict{ String,Array{Int64,1} } = Dict()
William F Godoy's avatar
William F Godoy committed
185 186 187 188 189
  
  nlevels::Int64 =  _find_max_nlevels(outputDirs)
  nranks::Int64 = _find_max_nranks(outputDirs)
  
  for level in (1:nlevels)   
William F Godoy's avatar
William F Godoy committed
190 191
    levelKey::String = string("Level_",level-1)
    push!(Yd_Levels, levelKey => [])
William F Godoy's avatar
William F Godoy committed
192
    println("Yd_Levels", Yd_Levels, " ", level)
William F Godoy's avatar
William F Godoy committed
193
    
William F Godoy's avatar
William F Godoy committed
194
    for rank in (1:nranks)
William F Godoy's avatar
William F Godoy committed
195 196
      rankKey::String = string(levelKey,"/Cell_D_", lpad(rank-1,5,"0") )
      push!(Yd_Ranks, rankKey => [])
William F Godoy's avatar
William F Godoy committed
197
    end
William F Godoy's avatar
William F Godoy committed
198
    
William F Godoy's avatar
William F Godoy committed
199 200 201
  end
  
  println("Yd_Ranks: ", Yd_Ranks)
202 203
    
  caseIDs::Array{String,1} = []
William F Godoy's avatar
William F Godoy committed
204
  caseDirs::Array{String,1} = []
205 206 207 208 209 210 211 212 213 214 215 216 217 218
      
  for (index,outputDir) in enumerate(outputDirs)
    
    inputFile::String = _get_input_file(extractor, outputDir)
    parameters = _input_parser(extractor, inputFile)
    rootPlotName = get(parameters, "amr.plot_file", "")

    # find all directories with rootPlotName and get its size
    # root of _plt directories 
    plotFileDirs = helper_get_prefix_directories(string(outputDir, "/", rootPlotName))

    counter::Int32 = 1
    
    # extract ncells
William F Godoy's avatar
William F Godoy committed
219
    ncells = X[!,"amr.ncells"][index]
220 221 222 223 224 225 226 227
    
    # caseID
    caseID = helper_get_relative_path(outputDir)
    
    for plotFileDir in plotFileDirs
      # Xd
      push!( Xd, counter*ncells)
      
William F Godoy's avatar
William F Godoy committed
228 229
      # Yd timesteps
      _push_datasize!(plotFileDir, Yd_Timesteps)
230
      
William F Godoy's avatar
William F Godoy committed
231 232
      levelDirs = helper_get_prefix_directories(string(plotFileDir,"/Level_"))
      currentLevels = helper_get_relative_path.(levelDirs)
233
      
William F Godoy's avatar
William F Godoy committed
234
      println("Current levels: ", currentLevels)
235
      
William F Godoy's avatar
William F Godoy committed
236 237
      # Levels
      for (levelKey,levelSizeValues) in Yd_Levels
William F Godoy's avatar
William F Godoy committed
238 239

        levelDir = string(plotFileDir,"/",levelKey)
William F Godoy's avatar
William F Godoy committed
240 241 242 243 244 245 246 247
        foundArray = findall( x -> x == levelKey, currentLevels)
        
        if size(foundArray,1) == 1
           _push_datasize!(levelDir, Yd_Levels[levelKey])
        else
           push!(Yd_Levels[levelKey],0)
        end
        
William F Godoy's avatar
William F Godoy committed
248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267
        # Level Rank data
        # add ranks 
        rankFullFiles = helper_get_prefix_directories(string(levelDir,"/Cell_D_"))
        rankFiles = helper_get_relative_path.(rankFullFiles)
           
        for rankID in 1:nranks
            
            rankKey::String = string(levelKey,"/Cell_D_", lpad(rankID-1,5,"0") )
            # check if file exist
            checkFileName =  string(levelDir,"/Cell_D_", lpad(rankID-1,5,"0") )
            
            if isfile( checkFileName )
              rankFileSize = filesize(checkFileName)
              push!(Yd_Ranks[rankKey], rankFileSize)
            else
              push!(Yd_Ranks[rankKey],0)  
            end
        end
        
      end
William F Godoy's avatar
William F Godoy committed
268
        
269
      # caseID
William F Godoy's avatar
William F Godoy committed
270 271 272 273
      push!(caseIDs, caseID )
      
      # caseDir
      push!(caseDirs, helper_get_relative_path(plotFileDir) )
274 275 276 277 278 279
      
      counter = counter + 1
    end
    
  end

280 281
  df::DataFrames.DataFrame = 
      DataFrames.DataFrame( plots_cells = Xd, 
William F Godoy's avatar
William F Godoy committed
282 283 284 285
                            plots_size = Yd_Timesteps)
                            
  for (key,Yd_Level) in Yd_Levels
    df[!,key] = Yd_Level
William F Godoy's avatar
William F Godoy committed
286 287 288 289 290
  end
  
  for (key,Yd_Rank) in Yd_Ranks
    df[!,key] = Yd_Rank
  end
William F Godoy's avatar
William F Godoy committed
291 292 293 294
  
  df[!,"caseID"] = caseIDs
  df[!,"caseDir"] = caseDirs

295 296 297 298 299 300 301
  println(df)
  
  CSV.write("plot_size.csv",  df, header=true)
  #display(Plots.scatter(Xd, Yd))
  #readline()
end