Commit 22a3d81a authored by Mostofa Patwary's avatar Mostofa Patwary
Browse files

removed the function compile_helper

parent f7d96a69

megatron/data/biencoder_dataset_utils.py

+0 −4
Original line number Diff line number Diff line
@@ -125,10 +125,6 @@ def get_block_samples_mapping(block_dataset, title_dataset, data_prefix, num_epo 
        print_rank_0(' > building samples index mapping for {} ...'.format(

            name))



        # compile/bind the C++ helper code

        from megatron.data.dataset_utils import compile_helper

        compile_helper()



        from megatron.data import helpers

        mapping_array = helpers.build_blocks_mapping(

            block_dataset.doc_idx,