fix ia3 mlp infused adapter (#8597)

Signed-off-by: Chen Cui <chcui@nvidia.com>
NVIDIA · Mar 6, 2024 · 7d0d39a · 7d0d39a
1 parent c945058
commit 7d0d39a
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py b/nemo/collections/nlp/modules/common/megatron/adapters/mcore_mixins.py
@@ -277,6 +277,10 @@ def glu(x):
             else:
                 intermediate_parallel = self.activation_func(intermediate_parallel)
 
+        infused_adapter = self.get_adapter_module(AdapterName.MLP_INFUSED)
+        if infused_adapter:
+            intermediate_parallel = infused_adapter(intermediate_parallel)
+
         # [s, b, h]
         output, output_bias = self.linear_fc2(intermediate_parallel)