Commit 2b47c5e7 authored by Raul Puri's avatar Raul Puri
Browse files

mmap sample map

parent cfe35fd3

megatron/data/bert_dataset.py

+1 −1
Original line number Diff line number Diff line
@@ -279,7 +279,7 @@ def get_samples_mapping_(indexed_dataset, 
    print_rank_0(' > loading indexed mapping from {}'.format(

        indexmap_filename))

    start_time = time.time()

    samples_mapping = np.load(indexmap_filename, allow_pickle=True)

    samples_mapping = np.load(indexmap_filename, allow_pickle=True, mmap_mode='r')

    print_rank_0('    loaded indexed file in {:3.3f} seconds'.format(

        time.time() - start_time))

    print_rank_0('    total number of samples: {}'.format(