Commit 6fbc0192 authored by Hines, Jesse's avatar Hines, Jesse
Browse files

Update ingests to use local disk

parent bb1c0913

docker-compose.yml

+4 −4
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ services: 
    volumes:

      - druid_shared:/opt/shared

      - coordinator_var:/opt/druid/var

      # - ./data:/data

      - ./data:/data

    depends_on:

      - zookeeper

      - postgres
@@ -58,7 +58,7 @@ services: 
    container_name: druid-broker

    volumes:

      - broker_var:/opt/druid/var

      # - ./data:/data

      - ./data:/data

    depends_on:

      - zookeeper

      - postgres
@@ -80,7 +80,7 @@ services: 
    volumes:

      - druid_shared:/opt/shared

      - historical_var:/opt/druid/var

      # - ./data:/data

      - ./data:/data

    depends_on: 

      - zookeeper

      - postgres
@@ -102,7 +102,7 @@ services: 
    volumes:

      - druid_shared:/opt/shared

      - middle_var:/opt/druid/var

      # - ./data:/data

      - ./data:/data

    depends_on: 

      - zookeeper

      - postgres

druid-environment.txt

+1 −1
Original line number Diff line number Diff line
@@ -39,4 +39,4 @@ druid_processing_numMergeBuffers=2 


DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>



druid_worker_capacity=7
 
druid_worker_capacity=10

druid_ingests/data-fugaku.json

+12 −7
Original line number Diff line number Diff line
@@ -3,12 +3,17 @@ 
    "spec": {

        "ioConfig": {

            "type": "index_parallel",

            // "inputSource": {

            //     "type": "s3",

            //     "objectGlob": "**.parquet",

            //     "prefixes": [

            //         "s3://scratch/raps-datasets/fugaku/"

            //     ]

            // },

            "inputSource": {

                "type": "s3",

                "objectGlob": "**.parquet",

                "prefixes": [

                    "s3://scratch/raps-datasets/fugaku/"

                ]

                "type": "local",

                "baseDir": "/data/fugaku/",

                "filter": "*.parquet"

            },

            "inputFormat": {

                "type": "parquet"
@@ -19,8 +24,8 @@ 
            "partitionsSpec": {

                "type": "dynamic"

            },

            "maxNumConcurrentSubTasks": 6,

            "maxRowsInMemory": 250000

            "maxNumConcurrentSubTasks": 2,

            "maxRowsInMemory": 100000

        },

        "dataSchema": {

            "dataSource": "svc-ts-exadigit-data-fugaku",

druid_ingests/data-lassen-allocation-history.json

+11 −5
Original line number Diff line number Diff line
@@ -3,11 +3,16 @@ 
    "spec": {

        "ioConfig": {

            "type": "index_parallel",

            // "inputSource": {

            //     "type": "s3",

            //     "prefixes": [

            //         "s3://scratch/raps-datasets/lassen/final_csm_allocation_history_hashed.csv"

            //     ]

            // },

            "inputSource": {

                "type": "s3",

                "prefixes": [

                    "s3://scratch/raps-datasets/lassen/final_csm_allocation_history_hashed.csv"

                ]

                "type": "local",

                "baseDir": "/data/lassen/final_csm_allocation_history_hashed.csv",

                "filter": "*"

            },

            "inputFormat": {

                "type": "csv",
@@ -19,7 +24,8 @@ 
            "partitionsSpec": {

                "type": "dynamic"

            },

            "maxNumConcurrentSubTasks": 4

            "maxNumConcurrentSubTasks": 2,

            "maxRowsInMemory": 100000

        },

        "dataSchema": {

            "dataSource": "svc-ts-exadigit-data-lassen-allocation-history",

druid_ingests/data-lassen-node-history.json

+11 −5
Original line number Diff line number Diff line
@@ -3,11 +3,16 @@ 
    "spec": {

        "ioConfig": {

            "type": "index_parallel",

            // "inputSource": {

            //     "type": "s3",

            //     "prefixes": [

            //         "s3://scratch/raps-datasets/lassen/final_csm_allocation_node_history_with_time.csv"

            //     ]

            // },

            "inputSource": {

                "type": "s3",

                "prefixes": [

                    "s3://scratch/raps-datasets/lassen/final_csm_allocation_node_history_with_time.csv"

                ]

                "type": "local",

                "baseDir": "/data/lassen/final_csm_allocation_node_history_with_time.csv",

                "filter": "*"

            },

            "inputFormat": {

                "type": "csv",
@@ -19,7 +24,8 @@ 
            "partitionsSpec": {

                "type": "dynamic"

            },

            "maxNumConcurrentSubTasks": 4

            "maxNumConcurrentSubTasks": 2,

            "maxRowsInMemory": 100000

        },

        "dataSchema": {

            "dataSource": "svc-ts-exadigit-data-lassen-node-history",