remove mcore-dist-opt (for now) (NVIDIA#323)

* remove mcore-dist-opt (for now) Signed-off-by: Alexandros Koumparoulis <akoumparouli@nvidia.com> * use EP=1 since we use APEX dist opt Signed-off-by: Alexandros Koumparoulis <akoumparouli@nvidia.com> --------- Signed-off-by: Alexandros Koumparoulis <akoumparouli@nvidia.com>
gdengk · May 15, 2024 · 1f42205 · 1f42205
1 parent 0632bb8
commit 1f42205
Showing 1 changed file with 4 additions and 6 deletions.
diff --git a/launcher_scripts/conf/training/mixtral/mixtral_8x7b.yaml b/launcher_scripts/conf/training/mixtral/mixtral_8x7b.yaml
@@ -52,9 +52,9 @@ model:
   micro_batch_size: 1
   global_batch_size: 256
   rampup_batch_size: null
-  tensor_model_parallel_size: 2
-  pipeline_model_parallel_size: 1
-  expert_model_parallel_size: 8
+  tensor_model_parallel_size: 8
+  pipeline_model_parallel_size: 4
+  expert_model_parallel_size: 1
   virtual_pipeline_model_parallel_size: null
   encoder_seq_length: 4096
   max_position_embeddings: 32768
@@ -145,9 +145,7 @@ model:
     - 0
     gen_shape: false
   optim:
-    name: mcore_distributed_optim
-    overlap_grad_sync: true
-    overlap_param_sync: true
+    name: distributed_fused_adam
     lr: 0.0001
     weight_decay: 0.1
     betas: