Commit 2c247afb authored by Jared Casper's avatar Jared Casper
Browse files

Add arg needed for tokenizer.

parent 37475717

tools/preprocess_data.py

+1 −0
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ def get_args(): 
    args.rank = 0

    args.make_vocab_size_divisible_by = 128

    args.tensor_model_parallel_size = 1

    args.vocab_extra_ids = 0



    return args