Commit 050c24d5 authored by Steven Steinke's avatar Steven Steinke
Browse files

Merge branch 'staging_docs' into 'staging' 

Staging docs

See merge request ADLR/megatron-lm!64
parents d928dd51 3df23c72

README.md

+438 −187

File changed.

Preview size limit exceeded, changes collapsed.

examples/evaluate_zeroshot_gpt2.sh

0 → 100755
+38 −0
Original line number Diff line number Diff line 
#!/bin/bash



WORLD_SIZE=8



DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \

                  --nnodes 1 \

                  --node_rank 0 \

                  --master_addr localhost \

                  --master_port 6000"



TASK="LAMBADA"



VALID_DATA=<lambada path>

VOCAB_FILE=gpt2-vocab.json

MERGE_FILE=gpt2-merges.txt

CHECKPOINT=checkpoints/gpt2_345m





python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \

               --task $TASK \

               --valid-data $VALID_DATA \

               --tokenizer-type GPT2BPETokenizer \

               --strict-lambada

               --vocab-file $VOCAB_FILE \

               --merge-file $MERGE_FILE \

               --load $CHECKPOINT \

               --model-parallel-size 1 \

               --num-layers 24 \

               --hidden-size 1024 \

               --num-attention-heads 16 \

               --batch-size 8 \

               --checkpoint-activations \

               --seq-length 512 \

               --max-position-embeddings 512 \

               --log-interval 10 \

               --fp16 \

               --no-load-optim \

               --no-load-rng

examples/finetine_race_distributed.sh

0 → 100755
+47 −0
Original line number Diff line number Diff line 
#!/bin/bash



WORLD_SIZE=8



DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \

                  --nnodes 1 \

                  --node_rank 0 \

                  --master_addr localhost \

                  --master_port 6000"



TRAIN_DATA="data/RACE/train/middle"

VALID_DATA="data/RACE/dev/middle \

            data/RACE/dev/high"

VOCAB_FILE=bert-vocab.txt

PRETRIANED_CHECKPOINT=checkpoints/bert_345m

CHECKPOINT_PATH=checkpoints/bert_345m_race



python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \

               --task RACE \

               --seed 1234 \

               --train-data $TRAIN_DATA \

               --valid-data $VALID_DATA \

               --tokenizer-type BertWordPieceLowerCase \

               --vocab-file $VOCAB_FILE \

               --epochs 3 \

               --pretrained-checkpoint $PRETRIANED_CHECKPOINT \

               --model-parallel-size 1 \

               --num-layers 24 \

               --hidden-size 1024 \

               --num-attention-heads 16 \

               --batch-size 4 \

               --checkpoint-activations \

               --lr 1.0e-5 \

               --lr-decay-style linear \

               --warmup 0.06 \

               --seq-length 512 \

               --max-position-embeddings 512 \

               --save-interval 500000 \

               --save $CHECKPOINT_PATH \

               --log-interval 10 \

               --eval-interval 100 \

               --eval-iters 50 \

               --weight-decay 1.0e-1 \

               --clip-grad 1.0 \

               --hidden-dropout 0.1 \

               --attention-dropout 0.1 \

               --fp16

examples/finetune_mnli_distributed.sh

0 → 100755
+44 −0
Original line number Diff line number Diff line 
#!/bin/bash



WORLD_SIZE=8



DISTRIBUTED_ARGS="--nproc_per_node $WORLD_SIZE \

                  --nnodes 1 \

                  --node_rank 0 \

                  --master_addr localhost \

                  --master_port 6000"



TRAIN_DATA="data/glue_data/MNLI/train.tsv"

VALID_DATA="data/glue_data/MNLI/dev_matched.tsv \

            data/glue_data/MNLI/dev_mismatched.tsv"

PRETRAINED_CHECKPOINT=checkpoints/bert_345m

VOCAB_FILE=bert-vocab.txt

CHECKPOINT_PATH=checkpoints/bert_345m_mnli



python -m torch.distributed.launch $DISTRIBUTED_ARGS ./tasks/main.py \

               --task MNLI \

               --seed 1234 \

               --train-data $TRAIN_DATA \

               --valid-data $VALID_DATA \

               --tokenizer-type BertWordPieceLowerCase \

               --vocab-file $VOCAB_FILE \

               --epochs 5 \

               --pretrained-checkpoint $PRETRAINED_CHECKPOINT \

               --model-parallel-size 1 \

               --num-layers 24 \

               --hidden-size 1024 \

               --num-attention-heads 16 \

               --batch-size 8 \

               --checkpoint-activations \

               --lr 5.0e-5 \

               --lr-decay-style linear \

               --warmup 0.065 \

               --seq-length 512 \

               --max-position-embeddings 512 \

               --save-interval 500000 \

               --save $CHECKPOINT_PATH \

               --log-interval 10 \

               --eval-interval 100 \

               --eval-iters 50 \

               --weight-decay 1.0e-1 \

               --fp16

examples/generate_text.sh

+17 −24
Original line number Diff line number Diff line 
#!/bin/bash



CHECKPOINT_PATH=checkpoints/gpt2_345m/

MPSIZE=1

NLAYERS=12

NHIDDEN=768

NATT=12

MAXSEQLEN=1024

CHECKPOINT_PATH=checkpoints/gpt2_345m

VOCAB_FILE=gpt2-vocab.json

MERGE_FILE=gpt2-merges.txt



#SAMPLING ARGS

TEMP=0.9

#If TOPK/TOPP are 0 it defaults to greedy sampling, top-k will also override top-p

TOPK=0

TOPP=0



python generate_samples.py \

       --model-parallel-size $MPSIZE \

       --num-layers $NLAYERS \

       --hidden-size $NHIDDEN \

python tools/generate_samples_gpt2.py \

       --model-parallel-size 1 \

       --num-layers 24 \

       --hidden-size 1024 \

       --load $CHECKPOINT_PATH \

       --num-attention-heads $NATT \

       --num-attention-heads 16 \

       --max-position-embeddings 1024 \

       --tokenizer-type GPT2BPETokenizer \

       --fp16 \

       --cache-dir cache \

       --out-seq-length $MAXSEQLEN \

       --temperature $TEMP \

       --top_k $TOPK \

       --genfile dbg_unconditional.json \

       --num-samples 10 \

       --top_p $TOPP \

       --batch-size 2 \

       --seq-length 1024 \

       --out-seq-length 1024 \

       --temperature 1.0 \

       --vocab-file $VOCAB_FILE \

       --merge-file $MERGE_FILE \

       --genfile unconditional_samples.json \

       --num-samples 2 \

       --top_p 0.9 \

       --recompute