Commit 4bea4e76 authored by Mostofa Patwary's avatar Mostofa Patwary
Browse files

Merge branch 'main_retriver_merge' into main_retriver_merge_ict_eval

parents 613d0fe8 22a3d81a

megatron/data/biencoder_dataset_utils.py

+0 −4
Original line number Diff line number Diff line
@@ -125,10 +125,6 @@ def get_block_samples_mapping(block_dataset, title_dataset, data_prefix, num_epo 
        print_rank_0(' > building samples index mapping for {} ...'.format(

            name))



        # compile/bind the C++ helper code

        from megatron.data.dataset_utils import compile_helper

        compile_helper()



        from megatron.data import helpers

        mapping_array = helpers.build_blocks_mapping(

            block_dataset.doc_idx,