Commit c3e8abe2 authored by Hines, Jesse's avatar Hines, Jesse
Browse files

Add adastraMI250 download

parent ffb2d1b4

.gitignore

+1 −0
Original line number Diff line number Diff line
@@ -9,3 +9,4 @@ models/fmu-models 
.shell-completion-cache

raps-output-*

ppo_raps_logs

/data

raps/dataloaders/adastraMI250.py

+25 −15
Original line number Diff line number Diff line 
"""



# get the data

    Download `AdastaJobsMI250_15days.parquet` from

    https://zenodo.org/records/14007065/files/AdastaJobsMI250_15days.parquet



```

raps download --system adastraMI250

```

This will download the dataset from https://zenodo.org/records/14007065/files/AdastaJobsMI250_15days.parquet



# to simulate the dataset

raps run -f /path/to/AdastaJobsMI250_15days.parquet --system adastraMI250
@@ -17,12 +17,14 @@ 


# to analyze dataset

python -m raps.telemetry -f /path/to/AdastaJobsMI250_15days.parquet --system adastraMI250 -v



"""

import uuid

import numpy as np

import pandas as pd

from pathlib import Path

from datetime import datetime

from tqdm import tqdm

import urllib.request



from ..job import job_dict, Job

from ..utils import WorkloadData
@@ -279,3 +281,11 @@ def cdu_pos(index: int, config: dict) -> tuple[int, int]: 
    name = CDU_NAMES[index - 1]

    row, col = int(name[2]), int(name[3:5])

    return (row, col)





def download(dest: Path, start: datetime | None, end: datetime | None):

    dest.mkdir(parents = True)

    filename = "AdastaJobsMI250_15days.parquet"

    print(f"Downloading {filename}")

    urllib.request.urlretrieve(f"https://zenodo.org/records/14007065/files/{filename}", dest / filename)

    print("Done!")

raps/dataloaders/fugaku.py

+1 −1
Original line number Diff line number Diff line 
"""

Use the fugaku dataset published at https://zenodo.org/records/11467483

Uses the fugaku dataset published at https://zenodo.org/records/11467483



Note that F-Data doesn't give a list of nodes used, so we set 'scheduled_nodes' to None

which triggers the scheduler to schedule the nodes itself.

raps/dataloaders/lassen.py

+6 −7
Original line number Diff line number Diff line
@@ -10,23 +10,22 @@ Reference: 


Usage Instructions:



    git clone https://github.com/LLNL/LAST/ && cd LAST

    git lfs pull

    raps download --system lassen



    # to analyze dataset and plot histograms

    raps telemetry -f /path/to/LAST/Lassen-Supercomputer-Job-Dataset --system lassen --plot

    raps telemetry -f ./data/lassen/Lassen-Supercomputer-Job-Dataset --system lassen --plot



    # to simulate the dataset as submitted

    raps run -f /path/to/LAST/Lassen-Supercomputer-Job-Dataset --system lassen

    raps run -f ./data/lassen/Lassen-Supercomputer-Job-Dataset --system lassen



    # to modify the submit times of the telemetry according to Poisson distribution

    raps run -f /path/to/LAST/Lassen-Supercomputer-Job-Dataset --system lassen --arrival poisson

    raps run -f ./data/lassen/Lassen-Supercomputer-Job-Dataset --system lassen --arrival poisson



    # to fast-forward 365 days and replay for 1 day. This region day has 2250 jobs with 1650 jobs executed.

    raps run -f /path/to/LAST/Lassen-Supercomputer-Job-Dataset --system lassen --start '2019-08-22T00:00:00+00:00' -t 1d

    raps run -f ./data/lassen/Lassen-Supercomputer-Job-Dataset --system lassen --start '2019-08-22T00:00:00+00:00' -t 1d



    # For the network replay this command gives suiteable snapshots:

    raps run -f /path/to/LAST/Lassen-Supercomputer-Job-Dataset --system lassen --policy fcfs --backfill firstfit -t 12h --arrival poisson  # noqa

    raps run -f ./data/lassen/Lassen-Supercomputer-Job-Dataset --system lassen --policy fcfs --backfill firstfit -t 12h --arrival poisson  # noqa



"""

import math

raps/dataloaders/marconi100.py

+5 −1
Original line number Diff line number Diff line
@@ -6,7 +6,11 @@ of The International Conference on High Performance Computing, 
Network, Storage, and Analysis. 2023.



# get the data

Download `job_table.parquet` from https://zenodo.org/records/10127767

Download the dataset with

```

raps download --system marconi100

```

This will download the dataset from https://zenodo.org/records/10127767



# to simulate the dataset

raps run -f /path/to/job_table.parquet --system marconi100

raps/dataloaders/frontier.py

+7 −7

File changed.

Contains only whitespace changes.