Add ingests for other system data (5b35c015) · Commits · ExaDigiT / sim-server

druid_ingests/data-fugaku.json

0 → 100644

+280 −0

Original line number	Diff line number	Diff line
		{
		"type": "index_parallel",
		"spec": {
		"ioConfig": {
		"type": "index_parallel",
		"inputSource": {
		"type": "s3",
		"objectGlob": "**.parquet",
		"prefixes": [
		"s3://scratch/raps-datasets/fugaku/"
		]
		},
		"inputFormat": {
		"type": "parquet"
		}
		},
		"tuningConfig": {
		"type": "index_parallel",
		"partitionsSpec": {
		"type": "dynamic"
		},
		"maxNumConcurrentSubTasks": 6,
		"maxRowsInMemory": 250000
		},
		"dataSchema": {
		"dataSource": "svc-ts-exadigit-data-fugaku",
		"timestampSpec": {
		"column": "!!!_no_such_column_!!!",
		"missingValue": "2010-01-01T00:00:00Z"
		},
		"transformSpec": {
		"transforms": [
		{
		"name": "__time",
		"type": "expression",
		"expression": "timestamp_parse(sdt)"
		},
		{
		"name": "adt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(adt))"
		},
		{
		"name": "qdt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(qdt))"
		},
		{
		"name": "schedsdt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(schedsdt))"
		},
		{
		"name": "deldt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(deldt))"
		},
		{
		"name": "sdt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(sdt))"
		},
		{
		"name": "edt",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(edt))"
		}
		]
		},
		"dimensionsSpec": {
		"dimensions": [
		{
		"type": "string",
		"name": "jid"
		},
		{
		"type": "string",
		"name": "usr"
		},
		{
		"type": "string",
		"name": "jnam"
		},
		{
		"type": "long",
		"name": "cnumr"
		},
		{
		"type": "long",
		"name": "cnumat"
		},
		{
		"type": "long",
		"name": "cnumut"
		},
		{
		"type": "long",
		"name": "nnumr"
		},
		{
		"type": "string",
		"name": "adt"
		},
		{
		"type": "string",
		"name": "qdt"
		},
		{
		"type": "string",
		"name": "schedsdt"
		},
		{
		"type": "string",
		"name": "deldt"
		},
		{
		"type": "long",
		"name": "ec"
		},
		{
		"type": "double",
		"name": "elpl"
		},
		{
		"type": "string",
		"name": "sdt"
		},
		{
		"type": "string",
		"name": "edt"
		},
		{
		"type": "long",
		"name": "nnuma"
		},
		{
		"type": "double",
		"name": "idle_time_ave"
		},
		{
		"type": "long",
		"name": "nnumu"
		},
		{
		"type": "long",
		"name": "pri"
		},
		{
		"type": "string",
		"name": "jobenv_req"
		},
		{
		"type": "string",
		"name": "pclass"
		},
		{
		"type": "string",
		"name": "exit state"
		},
		{
		"type": "double",
		"name": "duration"
		}
		]
		},
		"metricsSpec": [
		{
		"type": "doubleMax",
		"name": "perf1",
		"fieldName": "perf1"
		},
		{
		"type": "doubleMax",
		"name": "perf2",
		"fieldName": "perf2"
		},
		{
		"type": "doubleMax",
		"name": "perf3",
		"fieldName": "perf3"
		},
		{
		"type": "doubleMax",
		"name": "perf4",
		"fieldName": "perf4"
		},
		{
		"type": "doubleMax",
		"name": "perf5",
		"fieldName": "perf5"
		},
		{
		"type": "doubleMax",
		"name": "perf6",
		"fieldName": "perf6"
		},
		{
		"type": "doubleMax",
		"name": "mszl",
		"fieldName": "mszl"
		},
		{
		"type": "doubleMax",
		"name": "econ",
		"fieldName": "econ"
		},
		{
		"type": "doubleMax",
		"name": "avgpcon",
		"fieldName": "avgpcon"
		},
		{
		"type": "doubleMax",
		"name": "minpcon",
		"fieldName": "minpcon"
		},
		{
		"type": "doubleMax",
		"name": "maxpcon",
		"fieldName": "maxpcon"
		},
		{
		"type": "doubleMax",
		"name": "msza",
		"fieldName": "msza"
		},
		{
		"type": "doubleMax",
		"name": "mmszu",
		"fieldName": "mmszu"
		},
		{
		"type": "doubleMax",
		"name": "uctmut",
		"fieldName": "uctmut"
		},
		{
		"type": "doubleMax",
		"name": "sctmut",
		"fieldName": "sctmut"
		},
		{
		"type": "doubleMax",
		"name": "usctmut",
		"fieldName": "usctmut"
		},
		{
		"type": "longMax",
		"name": "freq_req",
		"fieldName": "freq_req"
		},
		{
		"type": "longMax",
		"name": "freq_alloc",
		"fieldName": "freq_alloc"
		},
		{
		"type": "doubleMax",
		"name": "flops",
		"fieldName": "flops"
		},
		{
		"type": "doubleMax",
		"name": "mbwidth",
		"fieldName": "mbwidth"
		},
		{
		"type": "doubleMax",
		"name": "opint",
		"fieldName": "opint"
		}
		],
		"granularitySpec": {
		"queryGranularity": "none",
		"rollup": false,
		"segmentGranularity": "week"
		}
		}
		}
		}
		No newline at end of file

druid_ingests/data-lassen-allocation-history.json

0 → 100644

+133 −0

Original line number	Diff line number	Diff line
		{
		"type": "index_parallel",
		"spec": {
		"ioConfig": {
		"type": "index_parallel",
		"inputSource": {
		"type": "s3",
		"prefixes": [
		"s3://scratch/raps-datasets/lassen/final_csm_allocation_history_hashed.csv"
		]
		},
		"inputFormat": {
		"type": "csv",
		"findColumnsFromHeader": true
		}
		},
		"tuningConfig": {
		"type": "index_parallel",
		"partitionsSpec": {
		"type": "dynamic"
		},
		"maxNumConcurrentSubTasks": 4
		},
		"dataSchema": {
		"dataSource": "svc-ts-exadigit-data-lassen-allocation-history",
		"timestampSpec": {
		"column": "!!!_no_such_column_!!!",
		"missingValue": "2010-01-01T00:00:00Z"
		},
		"transformSpec": {
		"transforms": [
		{
		"name": "__time",
		"type": "expression",
		"expression": "timestamp_parse(begin_time)"
		},
		{
		"name": "begin_time",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(begin_time))"
		},
		{
		"name": "end_time",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(end_time))"
		},
		{
		"name": "job_submit_time",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(job_submit_time))"
		}
		]
		},
		"granularitySpec": {
		"queryGranularity": "none",
		"rollup": false,
		"segmentGranularity": "year"
		},
		"dimensionsSpec": {
		"dimensions": [
		{
		"type": "long",
		"name": "column_1"
		},
		{
		"type": "long",
		"name": "allocation_id"
		},
		{
		"type": "long",
		"name": "primary_job_id"
		},
		"launch_node_name",
		{
		"type": "long",
		"name": "isolated_cores"
		},
		{
		"type": "string",
		"name": "user_flags"
		},
		"system_flags",
		{
		"type": "long",
		"name": "ssd_min"
		},
		{
		"type": "long",
		"name": "ssd_max"
		},
		{
		"type": "long",
		"name": "num_nodes"
		},
		{
		"type": "long",
		"name": "num_processors"
		},
		{
		"type": "long",
		"name": "num_gpus"
		},
		{
		"type": "long",
		"name": "projected_memory"
		},
		"type",
		"job_type",
		"hashed_user_id",
		"hashed_user_group_id",
		"begin_time",
		"end_time",
		{
		"type": "long",
		"name": "exit_status"
		},
		"job_submit_time",
		"queue",
		"requeue",
		{
		"type": "long",
		"name": "time_limit"
		},
		"smt_mode",
		{
		"type": "string",
		"name": "core_blink"
		}
		]
		}
		}
		}
		}
		No newline at end of file

druid_ingests/data-lassen-node-history.json

0 → 100644

+100 −0

Original line number	Diff line number	Diff line
		{
		"type": "index_parallel",
		"spec": {
		"ioConfig": {
		"type": "index_parallel",
		"inputSource": {
		"type": "s3",
		"prefixes": [
		"s3://scratch/raps-datasets/lassen/final_csm_allocation_node_history_with_time.csv"
		]
		},
		"inputFormat": {
		"type": "csv",
		"findColumnsFromHeader": true
		}
		},
		"tuningConfig": {
		"type": "index_parallel",
		"partitionsSpec": {
		"type": "dynamic"
		},
		"maxNumConcurrentSubTasks": 4
		},
		"dataSchema": {
		"dataSource": "svc-ts-exadigit-data-lassen-node-history",
		"timestampSpec": {
		"column": "job_begin_time",
		"format": "auto"
		},
		"transformSpec": {},
		"dimensionsSpec": {
		"dimensions": [
		{
		"type": "long",
		"name": "column_1"
		},
		{
		"type": "long",
		"name": "allocation_id"
		},
		"node_name",
		"shared",
		{
		"type": "long",
		"name": "energy"
		},
		{
		"type": "long",
		"name": "gpfs_read"
		},
		{
		"type": "long",
		"name": "gpfs_write"
		},
		{
		"type": "long",
		"name": "ib_tx"
		},
		{
		"type": "long",
		"name": "ib_rx"
		},
		{
		"type": "long",
		"name": "power_cap"
		},
		{
		"type": "long",
		"name": "power_shifting_ratio"
		},
		{
		"type": "long",
		"name": "power_cap_hit"
		},
		{
		"type": "long",
		"name": "gpu_usage"
		},
		{
		"name": "gpu_energy",
		"type": "long"
		},
		{
		"type": "long",
		"name": "cpu_usage"
		},
		{
		"type": "long",
		"name": "memory_usage_max"
		}
		]
		},
		"granularitySpec": {
		"queryGranularity": "none",
		"rollup": false,
		"segmentGranularity": "year"
		}
		}
		}
		}
		No newline at end of file

druid_ingests/data-lassen-step-history.json

0 → 100644

+110 −0

Original line number	Diff line number	Diff line
		{
		"type": "index_parallel",
		"spec": {
		"ioConfig": {
		"type": "index_parallel",
		"inputSource": {
		"type": "s3",
		"prefixes": [
		"s3://scratch/raps-datasets/lassen/final_csm_step_history.csv"
		]
		},
		"inputFormat": {
		"type": "csv",
		"findColumnsFromHeader": true
		}
		},
		"tuningConfig": {
		"type": "index_parallel",
		"partitionsSpec": {
		"type": "dynamic"
		},
		"maxNumConcurrentSubTasks": 4
		},
		"dataSchema": {
		"dataSource": "svc-ts-exadigit-data-fugaku-lassen-step-history",
		"timestampSpec": {
		"column": "!!!_no_such_column_!!!",
		"missingValue": "2010-01-01T00:00:00Z"
		},
		"transformSpec": {
		"transforms": [
		{
		"name": "__time",
		"type": "expression",
		"expression": "timestamp_parse(begin_time)"
		},
		{
		"name": "begin_time",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(begin_time))"
		},
		{
		"name": "end_time",
		"type": "expression",
		"expression": "timestamp_format(timestamp_parse(end_time))"
		}
		]
		},
		"dimensionsSpec": {
		"dimensions": [
		{
		"type": "long",
		"name": "allocation_id"
		},
		{
		"type": "long",
		"name": "step_id"
		},
		"begin_time",
		"end_time",
		{
		"type": "long",
		"name": "num_nodes"
		},
		{
		"type": "long",
		"name": "num_processors"
		},
		{
		"type": "long",
		"name": "num_gpus"
		},
		{
		"type": "long",
		"name": "num_tasks"
		},
		{
		"type": "long",
		"name": "projected_memory"
		},
		{
		"type": "long",
		"name": "user_flags"
		},
		{
		"type": "long",
		"name": "exit_status"
		},
		{
		"type": "double",
		"name": "total_u_time"
		},
		{
		"type": "double",
		"name": "total_s_time"
		},
		{
		"type": "long",
		"name": "omp_thread_limit"
		}
		]
		},
		"granularitySpec": {
		"queryGranularity": "none",
		"rollup": false,
		"segmentGranularity": "month"
		}
		}
		}
		}
		No newline at end of file

druid_ingests/data-marconi100.json

0 → 100644

+158 −0

File added.

Preview size limit exceeded, changes collapsed.