Commit 8676baca authored by Rewon Child's avatar Rewon Child
Browse files

Update comments

parent 14c85e64

megatron/optimizer/clip_grads.py

+0 −2
Original line number Diff line number Diff line
@@ -138,10 +138,8 @@ def count_zeros_fp32(parameters): 
            grad = param.grad.detach()

            grads_to_count.append(grad)



    # Norm parameters.

    total_num_zeros = 0.0



    # Calculate norm.

    for grad in grads_to_count:

        num_zeros = grad.numel() - torch.count_nonzero(grad)

        total_num_zeros = num_zeros + total_num_zeros