Commit d7d788ae authored by mshoeybi's avatar mshoeybi
Browse files

Merge branch 'patch-1' of https://github.com/eqy/Megatron-LM into github_small_prs

parents 91fcbd68 eddf7593

megatron/mpu/initialize.py

+4 −0
Original line number Diff line number Diff line
@@ -356,9 +356,13 @@ def get_data_parallel_rank(): 


def destroy_model_parallel():

    """Set the groups to none."""

    global _MODEL_PARALLEL_GROUP

    _MODEL_PARALLEL_GROUP = None

    global _TENSOR_MODEL_PARALLEL_GROUP

    _TENSOR_MODEL_PARALLEL_GROUP = None

    global _PIPELINE_MODEL_PARALLEL_GROUP

    _PIPELINE_MODEL_PARALLEL_GROUP = None

    global _DATA_PARALLEL_GROUP

    _DATA_PARALLEL_GROUP = None

    global _EMBEDDING_GROUP

    _EMBEDDING_GROUP = None