Commit bba90f79 authored by Mohammad Shoeybi's avatar Mohammad Shoeybi
Browse files

Merge branch 'preprocess_fix' into 'main' 

Add arg needed for tokenizer.

See merge request ADLR/megatron-lm!272
parents 37475717 2c247afb

tools/preprocess_data.py

+1 −0
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ def get_args(): 
    args.rank = 0

    args.make_vocab_size_divisible_by = 128

    args.tensor_model_parallel_size = 1

    args.vocab_extra_ids = 0



    return args