Commit da1c96e9 authored by Deepak Narayanan's avatar Deepak Narayanan
Browse files

Add assertion for now preventing usage of pipeline parallelism with T5 model

parent 2babcaf6

megatron/model/language_model.py

+4 −0
Original line number Diff line number Diff line
@@ -331,6 +331,10 @@ class TransformerLanguageModel(MegatronModule): 
        # Decoder (usually set to False, True if part of an encoder-decoder

        # architecture and in decoder-only stage).

        if self.add_decoder:

            # Temporary assertion until we verify correctness of pipeline parallelism

            # implementation of T5.

            assert args.pipeline_model_parallel_size == 1, \

                'pipeline parallelism is not supported in the presence of decoder'

            self.decoder = ParallelTransformer(

                self.init_method,

                output_layer_init_method,