Commit 7ee23804 authored by Hines, Jesse's avatar Hines, Jesse
Browse files

Add lassen dataloader

parent 9032ce5e
Loading
Loading
Loading
Loading
+37 −0
Original line number Diff line number Diff line
@@ -133,8 +133,45 @@ def fetch_marconi100_data(sim_config: SimConfig, raps_config: dict):
    return jobs


def fetch_lassen_data(sim_config: SimConfig, raps_config: dict):
    druid_engine = get_druid_engine()
    start, end = sim_config.start, sim_config.end

    allocation_df = query_time_range(
        "svc-ts-exadigit-data-lassen-allocation-history", start, end, 'end_time',
        druid_engine = druid_engine,
        parse_dates = ["begin_time", "end_time", "job_submit_time"],
    )

    tbl = get_table("svc-ts-exadigit-data-lassen-node-history", druid_engine)
    node_query = (
        sqla.select(sqla.text("*"))
            .where(
                (tbl.c['__time'] <= to_timestamp(end)) &
                (tbl.c['__time'] >= to_timestamp(start - timedelta(days=3)))
            )
    )
    node_df = pd.read_sql(node_query, druid_engine)

    step_df = query_time_range(
        "svc-ts-exadigit-data-fugaku-lassen-step-history", start, end, 'end_time',
        druid_engine = druid_engine,
        parse_dates = ["begin_time", "end_time"],
    )

    telemetry = Telemetry(system = "lassen", config = raps_config)
    jobs = telemetry.load_data_from_df(
        allocation_df = allocation_df, node_df = node_df, step_df = step_df,
        min_time = start,
        reschedule = sim_config.scheduler.reschedule,
        config = raps_config,
    )
    return jobs


DATA_LOADERS = {
    "frontier": fetch_frontier_data,
    "fugaku": fetch_fugaku_data,
    "marconi100": fetch_marconi100_data,
    "lassen": fetch_lassen_data,
}
 No newline at end of file
Compare 879b2a49 to d8c397dc
Original line number Diff line number Diff line
Subproject commit 879b2a4927377d6f92e1faad097162245d6817bc
Subproject commit d8c397dcb81a6503475bf80f410ad09508311576