Loading pretrain_crusher_mol_lamb.slurm +22 −61 Original line number Diff line number Diff line #!/bin/bash #SBATCH -A MED106_crusher #SBATCH -N 16 #SBATCH -t 00:25:00 #SBATCH -N 1 #SBATCH -t 00:05:00 #SBATCH -J crusher #SBATCH -o %x-%j.out #SBATCH -p batch Loading @@ -10,7 +10,8 @@ set +x source /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda520/etc/profile.d/conda.sh conda activate /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda520 conda activate /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda530 # conda activate /gpfs/alpine/med106/world-shared/irl1/CRUSHER_ENV/conda520 module load PrgEnv-gnu module load gcc/10.3.0 Loading @@ -29,8 +30,8 @@ gpus_per_rank=$((8/$ranks_per_node)) ranks_total=$(($ranks_per_node*$SLURM_JOB_NUM_NODES)) # export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" # export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6.0.29 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" # export LD_PRELOAD="${LD_PRELOAD} ${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so" export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6.0.29 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" export LD_PRELOAD="${LD_PRELOAD} ${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so" # export LD_PRELOAD="$LD_PRELOAD /lib64/libtinfo.so.6 /lib64/libncurses.so.6 /gpfs/alpine/world-shared/bif136/rccl/build/librccl.so" # export NCCL_NET_GDR_LEVEL=4 Loading @@ -44,56 +45,16 @@ ranks_total=$(($ranks_per_node*$SLURM_JOB_NUM_NODES)) # export FI_LOG_LEVEL=info # export NCCL_NET_GDR_LEVEL=3 DATASET_0="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000000.json_smiles_document" DATASET_1="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000001.json_smiles_document" DATASET_2="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000002.json_smiles_document" DATASET_3="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000003.json_smiles_document" DATASET_4="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000004.json_smiles_document" DATASET_5="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000005.json_smiles_document" DATASET_6="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000006.json_smiles_document" DATASET_7="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000007.json_smiles_document" DATASET_8="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000008.json_smiles_document" DATASET_9="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000009.json_smiles_document" DATASET_10="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000010.json_smiles_document" DATASET_11="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000011.json_smiles_document" DATASET_12="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000012.json_smiles_document" DATASET_13="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000013.json_smiles_document" DATASET_14="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000014.json_smiles_document" DATASET_15="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000015.json_smiles_document" DATASET_16="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000016.json_smiles_document" DATASET_17="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000017.json_smiles_document" DATASET_18="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000018.json_smiles_document" DATASET_19="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000019.json_smiles_document" DATASET_20="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000020.json_smiles_document" DATASET_21="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000021.json_smiles_document" DATASET_22="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000022.json_smiles_document" DATASET_23="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000023.json_smiles_document" DATASET_24="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000024.json_smiles_document" DATASET_25="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000025.json_smiles_document" DATASET_26="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000026.json_smiles_document" DATASET_27="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000027.json_smiles_document" DATASET_28="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000028.json_smiles_document" DATASET_29="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000029.json_smiles_document" DATASET_30="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000030.json_smiles_document" DATASET_31="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000031.json_smiles_document" DATASET_32="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000032.json_smiles_document" DATASET_33="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000033.json_smiles_document" DATASET_34="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000034.json_smiles_document" DATASET_35="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000035.json_smiles_document" DATASET_36="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000036.json_smiles_document" DATASET_37="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000037.json_smiles_document" DATASET_38="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000038.json_smiles_document" DATASET_39="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000039.json_smiles_document" DATASET_40="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000040.json_smiles_document" DATASET_41="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000041.json_smiles_document" DATASET_42="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000042.json_smiles_document" DATASET_43="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000043.json_smiles_document" DATASET_44="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000044.json_smiles_document" DATASET_45="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000045.json_smiles_document" DATASET_46="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000046.json_smiles_document" DATASET_47="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000047.json_smiles_document" DATASET_48="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000048.json_smiles_document" DATASET_49="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000049.json_smiles_document" DATASET_0="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000000.json_smiles_document" DATASET_1="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000001.json_smiles_document" DATASET_2="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000002.json_smiles_document" DATASET_3="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000003.json_smiles_document" DATASET_4="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000004.json_smiles_document" DATASET_5="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000005.json_smiles_document" DATASET_6="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000006.json_smiles_document" DATASET_7="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000007.json_smiles_document" DATASET_8="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000008.json_smiles_document" DATASET_9="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000009.json_smiles_document" # DATASET="1.0 ${DATASET_0} 1.0 ${DATASET_1} 1.0 ${DATASET_2} 1.0 ${DATASET_3} 1.0 ${DATASET_4} 1.0 ${DATASET_5} 1.0 ${DATASET_6} 1.0 ${DATASET_7} 1.0 ${DATASET_8} 1.0 ${DATASET_9} 1.0 ${DATASET_10} 1.0 ${DATASET_11} 1.0 ${DATASET_12} 1.0 ${DATASET_13} 1.0 ${DATASET_14} 1.0 ${DATASET_15} 1.0 ${DATASET_16} 1.0 ${DATASET_17} 1.0 ${DATASET_18} 1.0 ${DATASET_19} 1.0 ${DATASET_20} 1.0 ${DATASET_21} 1.0 ${DATASET_22} 1.0 ${DATASET_23} 1.0 ${DATASET_24} 1.0 ${DATASET_25} 1.0 ${DATASET_26} 1.0 ${DATASET_27} 1.0 ${DATASET_28} 1.0 ${DATASET_29} 1.0 ${DATASET_30} 1.0 ${DATASET_31} 1.0 ${DATASET_32} 1.0 ${DATASET_33} 1.0 ${DATASET_34} 1.0 ${DATASET_35} 1.0 ${DATASET_36} 1.0 ${DATASET_37} 1.0 ${DATASET_38} 1.0 ${DATASET_39} 1.0 ${DATASET_40} 1.0 ${DATASET_41} 1.0 ${DATASET_42} 1.0 ${DATASET_43} 1.0 ${DATASET_44} 1.0 ${DATASET_45} 1.0 ${DATASET_46} 1.0 ${DATASET_47} 1.0 ${DATASET_48} 1.0 ${DATASET_49}" Loading @@ -102,9 +63,9 @@ DATASET="1.0 ${DATASET_0} 1.0 ${DATASET_1} 1.0 ${DATASET_2} 1.0 ${DATASET_3} 1.0 export VOCAB_FILE=/gpfs/alpine/world-shared/med106/blnchrd/models/bert_metrics_brackets/tokenizer/vocab.txt export CHECKPOINT_PATH=/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/temp2 # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh # python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 12 --hidden-size 768 --num-attention-heads 12 --optimizer lamb --micro-batch-size 256 --global-batch-size 12288 --seq-length 64 --max-position-embeddings 512 --train-iters 300 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 100 --save-interval 10000 --eval-interval 100 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " source export_DDP_vars.sh python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 12 --hidden-size 768 --num-attention-heads 12 --optimizer lamb --micro-batch-size 256 --global-batch-size 12288 --seq-length 64 --max-position-embeddings 512 --train-iters 300 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 100 --save-interval 10000 --eval-interval 100 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh Loading @@ -124,7 +85,7 @@ export CHECKPOINT_PATH=/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatr # python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 4 --num-layers 64 --hidden-size 4096 --num-attention-heads 64 --optimizer lamb --micro-batch-size 16 --global-batch-size 128 --seq-length 64 --max-position-embeddings 512 --train-iters 1200 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 20 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # 25.5B srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " source export_DDP_vars.sh python pretrain_bert.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 4 --num-layers 80 --hidden-size 5120 --num-attention-heads 80 --optimizer lamb --micro-batch-size 16 --global-batch-size 4096 --seq-length 64 --max-position-embeddings 512 --train-iters 5000 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 50 --eval-iters 20 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh # python pretrain_bert.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 4 --num-layers 80 --hidden-size 5120 --num-attention-heads 80 --optimizer lamb --micro-batch-size 16 --global-batch-size 4096 --seq-length 64 --max-position-embeddings 512 --train-iters 5000 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 50 --eval-iters 20 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" Loading
pretrain_crusher_mol_lamb.slurm +22 −61 Original line number Diff line number Diff line #!/bin/bash #SBATCH -A MED106_crusher #SBATCH -N 16 #SBATCH -t 00:25:00 #SBATCH -N 1 #SBATCH -t 00:05:00 #SBATCH -J crusher #SBATCH -o %x-%j.out #SBATCH -p batch Loading @@ -10,7 +10,8 @@ set +x source /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda520/etc/profile.d/conda.sh conda activate /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda520 conda activate /gpfs/alpine/world-shared/med106/gounley1/crusher2/conda530 # conda activate /gpfs/alpine/med106/world-shared/irl1/CRUSHER_ENV/conda520 module load PrgEnv-gnu module load gcc/10.3.0 Loading @@ -29,8 +30,8 @@ gpus_per_rank=$((8/$ranks_per_node)) ranks_total=$(($ranks_per_node*$SLURM_JOB_NUM_NODES)) # export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" # export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6.0.29 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" # export LD_PRELOAD="${LD_PRELOAD} ${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so" export LD_PRELOAD="/opt/cray/pe/gcc/11.2.0/snos/lib64/libstdc++.so.6.0.29 /gpfs/alpine/world-shared/bip214/rocm_smi_lib/build/rocm_smi/librocm_smi64.so" export LD_PRELOAD="${LD_PRELOAD} ${CRAY_MPICH_ROOTDIR}/gtl/lib/libmpi_gtl_hsa.so" # export LD_PRELOAD="$LD_PRELOAD /lib64/libtinfo.so.6 /lib64/libncurses.so.6 /gpfs/alpine/world-shared/bif136/rccl/build/librccl.so" # export NCCL_NET_GDR_LEVEL=4 Loading @@ -44,56 +45,16 @@ ranks_total=$(($ranks_per_node*$SLURM_JOB_NUM_NODES)) # export FI_LOG_LEVEL=info # export NCCL_NET_GDR_LEVEL=3 DATASET_0="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000000.json_smiles_document" DATASET_1="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000001.json_smiles_document" DATASET_2="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000002.json_smiles_document" DATASET_3="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000003.json_smiles_document" DATASET_4="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000004.json_smiles_document" DATASET_5="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000005.json_smiles_document" DATASET_6="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000006.json_smiles_document" DATASET_7="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000007.json_smiles_document" DATASET_8="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000008.json_smiles_document" DATASET_9="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000009.json_smiles_document" DATASET_10="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000010.json_smiles_document" DATASET_11="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000011.json_smiles_document" DATASET_12="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000012.json_smiles_document" DATASET_13="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000013.json_smiles_document" DATASET_14="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000014.json_smiles_document" DATASET_15="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000015.json_smiles_document" DATASET_16="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000016.json_smiles_document" DATASET_17="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000017.json_smiles_document" DATASET_18="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000018.json_smiles_document" DATASET_19="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000019.json_smiles_document" DATASET_20="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000020.json_smiles_document" DATASET_21="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000021.json_smiles_document" DATASET_22="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000022.json_smiles_document" DATASET_23="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000023.json_smiles_document" DATASET_24="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000024.json_smiles_document" DATASET_25="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000025.json_smiles_document" DATASET_26="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000026.json_smiles_document" DATASET_27="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000027.json_smiles_document" DATASET_28="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000028.json_smiles_document" DATASET_29="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000029.json_smiles_document" DATASET_30="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000030.json_smiles_document" DATASET_31="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000031.json_smiles_document" DATASET_32="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000032.json_smiles_document" DATASET_33="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000033.json_smiles_document" DATASET_34="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000034.json_smiles_document" DATASET_35="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000035.json_smiles_document" DATASET_36="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000036.json_smiles_document" DATASET_37="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000037.json_smiles_document" DATASET_38="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000038.json_smiles_document" DATASET_39="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000039.json_smiles_document" DATASET_40="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000040.json_smiles_document" DATASET_41="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000041.json_smiles_document" DATASET_42="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000042.json_smiles_document" DATASET_43="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000043.json_smiles_document" DATASET_44="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000044.json_smiles_document" DATASET_45="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000045.json_smiles_document" DATASET_46="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000046.json_smiles_document" DATASET_47="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000047.json_smiles_document" DATASET_48="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000048.json_smiles_document" DATASET_49="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data/part-000049.json_smiles_document" DATASET_0="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000000.json_smiles_document" DATASET_1="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000001.json_smiles_document" DATASET_2="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000002.json_smiles_document" DATASET_3="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000003.json_smiles_document" DATASET_4="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000004.json_smiles_document" DATASET_5="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000005.json_smiles_document" DATASET_6="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000006.json_smiles_document" DATASET_7="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000007.json_smiles_document" DATASET_8="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000008.json_smiles_document" DATASET_9="/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/preprocess_data2/part-000009.json_smiles_document" # DATASET="1.0 ${DATASET_0} 1.0 ${DATASET_1} 1.0 ${DATASET_2} 1.0 ${DATASET_3} 1.0 ${DATASET_4} 1.0 ${DATASET_5} 1.0 ${DATASET_6} 1.0 ${DATASET_7} 1.0 ${DATASET_8} 1.0 ${DATASET_9} 1.0 ${DATASET_10} 1.0 ${DATASET_11} 1.0 ${DATASET_12} 1.0 ${DATASET_13} 1.0 ${DATASET_14} 1.0 ${DATASET_15} 1.0 ${DATASET_16} 1.0 ${DATASET_17} 1.0 ${DATASET_18} 1.0 ${DATASET_19} 1.0 ${DATASET_20} 1.0 ${DATASET_21} 1.0 ${DATASET_22} 1.0 ${DATASET_23} 1.0 ${DATASET_24} 1.0 ${DATASET_25} 1.0 ${DATASET_26} 1.0 ${DATASET_27} 1.0 ${DATASET_28} 1.0 ${DATASET_29} 1.0 ${DATASET_30} 1.0 ${DATASET_31} 1.0 ${DATASET_32} 1.0 ${DATASET_33} 1.0 ${DATASET_34} 1.0 ${DATASET_35} 1.0 ${DATASET_36} 1.0 ${DATASET_37} 1.0 ${DATASET_38} 1.0 ${DATASET_39} 1.0 ${DATASET_40} 1.0 ${DATASET_41} 1.0 ${DATASET_42} 1.0 ${DATASET_43} 1.0 ${DATASET_44} 1.0 ${DATASET_45} 1.0 ${DATASET_46} 1.0 ${DATASET_47} 1.0 ${DATASET_48} 1.0 ${DATASET_49}" Loading @@ -102,9 +63,9 @@ DATASET="1.0 ${DATASET_0} 1.0 ${DATASET_1} 1.0 ${DATASET_2} 1.0 ${DATASET_3} 1.0 export VOCAB_FILE=/gpfs/alpine/world-shared/med106/blnchrd/models/bert_metrics_brackets/tokenizer/vocab.txt export CHECKPOINT_PATH=/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatron-LM/temp2 # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh # python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 12 --hidden-size 768 --num-attention-heads 12 --optimizer lamb --micro-batch-size 256 --global-batch-size 12288 --seq-length 64 --max-position-embeddings 512 --train-iters 300 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 100 --save-interval 10000 --eval-interval 100 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " source export_DDP_vars.sh python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 12 --hidden-size 768 --num-attention-heads 12 --optimizer lamb --micro-batch-size 256 --global-batch-size 12288 --seq-length 64 --max-position-embeddings 512 --train-iters 300 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 100 --save-interval 10000 --eval-interval 100 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh Loading @@ -124,7 +85,7 @@ export CHECKPOINT_PATH=/gpfs/alpine/world-shared/med106/gounley1/crusher2/Megatr # python pretrain_bert.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 4 --num-layers 64 --hidden-size 4096 --num-attention-heads 64 --optimizer lamb --micro-batch-size 16 --global-batch-size 128 --seq-length 64 --max-position-embeddings 512 --train-iters 1200 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 20 --eval-iters 10 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # 25.5B srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " source export_DDP_vars.sh python pretrain_bert.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 4 --num-layers 80 --hidden-size 5120 --num-attention-heads 80 --optimizer lamb --micro-batch-size 16 --global-batch-size 4096 --seq-length 64 --max-position-embeddings 512 --train-iters 5000 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 50 --eval-iters 20 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16" # srun -u -n $ranks_total -c 8 --gpus-per-task=$gpus_per_rank --gpu-bind=closest bash -c " # source export_DDP_vars.sh # python pretrain_bert.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 4 --num-layers 80 --hidden-size 5120 --num-attention-heads 80 --optimizer lamb --micro-batch-size 16 --global-batch-size 4096 --seq-length 64 --max-position-embeddings 512 --train-iters 5000 --save $CHECKPOINT_PATH --data-path $DATASET --vocab-file $VOCAB_FILE --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.001 --min-lr 0.00001 --lr-decay-style linear --lr-warmup-fraction .01 --weight-decay 1e-2 --clip-grad 1.0 --log-interval 20 --save-interval 10000 --eval-interval 50 --eval-iters 20 --bert-no-binary-head --num-workers 2 --HIP 1 --fp16"
