From 184b5bff39a203a8ee7fa6bdfa4df8e7b0d56f21 Mon Sep 17 00:00:00 2001 From: Wenwen Qu Date: Thu, 4 Jan 2024 15:10:21 +0800 Subject: [PATCH] avoid allreduce when num_expert=1 --- internlm/core/scheduler/no_pipeline_scheduler.py | 2 +- internlm/core/scheduler/pipeline_scheduler.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internlm/core/scheduler/no_pipeline_scheduler.py b/internlm/core/scheduler/no_pipeline_scheduler.py index 79a6f62..f6cabbc 100644 --- a/internlm/core/scheduler/no_pipeline_scheduler.py +++ b/internlm/core/scheduler/no_pipeline_scheduler.py @@ -122,7 +122,7 @@ class NonPipelineScheduler(BaseScheduler): self._call_hooks("after_criterion", loss) moe_loss = ( sum(moe_losses) * gpc.config.loss.moe_loss_coeff - if hasattr(gpc.config.model, "num_experts") + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1 else torch.tensor(0.0, device=torch.cuda.current_device(), dtype=gpc.config.model.get("dtype")) ) moe_loss /= scale_loss diff --git a/internlm/core/scheduler/pipeline_scheduler.py b/internlm/core/scheduler/pipeline_scheduler.py index 0398783..2c7e0df 100644 --- a/internlm/core/scheduler/pipeline_scheduler.py +++ b/internlm/core/scheduler/pipeline_scheduler.py @@ -308,7 +308,7 @@ class PipelineScheduler(BaseScheduler): moe_loss = ( sum(moe_losses) * gpc.config.loss.moe_loss_coeff - if hasattr(gpc.config.model, "num_experts") + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1 else torch.tensor(0.0, device=torch.cuda.current_device(), dtype=gpc.config.model.get("dtype")) ) moe_loss /= self.num_microbatches @@ -446,7 +446,7 @@ class PipelineScheduler(BaseScheduler): output, label = pack_return_tensors(return_tensors) if len(return_tensors) > 0 else (None, None) - if hasattr(gpc.config.model, "num_experts"): + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1: dist.all_reduce(accum_moe_loss, group=gpc.get_group(ParallelMode.PIPELINE)) if accum_loss is not None: @@ -650,7 +650,7 @@ class PipelineScheduler(BaseScheduler): output, label = pack_return_tensors(return_tensors) if len(return_tensors) > 0 else (None, None) - if hasattr(gpc.config.model, "num_experts"): + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1: dist.all_reduce(accum_moe_loss, group=gpc.get_group(ParallelMode.PIPELINE)) if accum_loss is not None: @@ -859,7 +859,7 @@ class InterleavedPipelineScheduler(PipelineScheduler): moe_loss = ( sum(moe_losses) * gpc.config.loss.moe_loss_coeff - if hasattr(gpc.config.model, "num_experts") + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1 else torch.tensor(0.0, device=torch.cuda.current_device(), dtype=gpc.config.model.get("dtype")) ) moe_loss /= self.num_microbatches @@ -1391,7 +1391,7 @@ class InterleavedPipelineScheduler(PipelineScheduler): else: output, label = (None, None) - if hasattr(gpc.config.model, "num_experts"): + if hasattr(gpc.config.model, "num_experts") and gpc.config.model.num_experts > 1: dist.all_reduce(self._accum_moe_loss, group=gpc.get_group(ParallelMode.PIPELINE)) accum_moe_loss = self._accum_moe_loss