update liger configs

anyscale · Nov 13, 2024 · e176331 · e176331
1 parent 0a2853f
commit e176331
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 0 deletions.
diff --git a/...tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512-mlflow.yaml b/...tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512-mlflow.yaml
@@ -50,6 +50,16 @@ logger:
 worker_resources:
   anyscale/accelerator_shape:4xA10G: 0.001
 
+# Liger kernel configuration
+liger_kernel:
+  enabled: True
+  kwargs:
+    rms_norm: False
+    rope: True
+    swiglu: True
+    cross_entropy: True
+    fused_linear_cross_entropy: False
+
 # Lora configuration
 lora_config:
   r: 8

diff --git a/...-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512-wandb.yaml b/...-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512-wandb.yaml
@@ -45,6 +45,16 @@ logger:
 worker_resources:
   anyscale/accelerator_shape:4xA10G: 0.001
 
+# Liger kernel configuration
+liger_kernel:
+  enabled: True
+  kwargs:
+    rms_norm: False
+    rope: True
+    swiglu: True
+    cross_entropy: True
+    fused_linear_cross_entropy: False
+
 # Lora configuration
 lora_config:
   r: 8

diff --git a/...s/fine-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512.yaml b/...s/fine-tune-llm_v2/training_configs/custom/meta-llama/Meta-Llama-3-8B/lora/4xA10-512.yaml
@@ -41,6 +41,7 @@ deepspeed:
 worker_resources:
   anyscale/accelerator_shape:4xA10G: 0.001
 
+# Liger kernel configuration
 liger_kernel:
   enabled: True
   kwargs: