Unverified Commit eddf7593 authored by eqy's avatar eqy Committed by GitHub
Browse files

Destroy more groups in `destroy_model_parallel` 

Some tests expect a clean model parallel slate and complain if a previous test left something behind; this change clears more variables that the tests complain about.
parent 2387ce01

megatron/mpu/initialize.py

+4 −0
Original line number Diff line number Diff line
@@ -356,9 +356,13 @@ def get_data_parallel_rank(): 


def destroy_model_parallel():

    """Set the groups to none."""

    global _MODEL_PARALLEL_GROUP

    _MODEL_PARALLEL_GROUP = None

    global _TENSOR_MODEL_PARALLEL_GROUP

    _TENSOR_MODEL_PARALLEL_GROUP = None

    global _PIPELINE_MODEL_PARALLEL_GROUP

    _PIPELINE_MODEL_PARALLEL_GROUP = None

    global _DATA_PARALLEL_GROUP

    _DATA_PARALLEL_GROUP = None

    global _EMBEDDING_GROUP

    _EMBEDDING_GROUP = None