Commit 3c709cba authored by Raul Puri's avatar Raul Puri
Browse files

Merge branch 'remove_local_ddp_bcast' into 'master' 

removed bcast from local ddp

See merge request ADLR/megatron-lm!72
parents 73f73b81 97e07d64

megatron/model/distributed.py

+0 −4
Original line number Diff line number Diff line
@@ -31,10 +31,6 @@ class DistributedDataParallel(MegatronModule): 


        self.module = module

        self.data_parallel_group = mpu.get_data_parallel_group()

        src_rank = mpu.get_model_parallel_rank()

        for p in self.module.parameters():

            if torch.is_tensor(p):

                dist.broadcast(p, src_rank, group=self.data_parallel_group)



        def allreduce_params(reduce_after=True, no_scale=False, fp32_allreduce=False):

            if(self.needs_reduction):