Add Llama-3-Instruct-8B-RainbowPO to AlpacaEval (#416)

* Add RainbowPO to AlpacaEval * change names * Add model path and configs
tatsu-lab · Nov 11, 2024 · 6976988 · 6976988
1 parent fccfd4e
commit 6976988
Show file tree

Hide file tree

Showing 6 changed files with 70,300 additions and 0 deletions.
diff --git a/results/Llama-3-Instruct-8B-RainbowPO/model_outputs.json b/results/Llama-3-Instruct-8B-RainbowPO/model_outputs.json
diff --git a/results/Llama-3-Instruct-8B-RainbowPO/weighted_alpaca_eval_gpt4_turbo/annotations.json b/results/Llama-3-Instruct-8B-RainbowPO/weighted_alpaca_eval_gpt4_turbo/annotations.json
diff --git a/...lpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv b/...lpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv
@@ -21,6 +21,7 @@ SPPO-Gemma-2-9B-It-PairRM,48.23404468746583,1.4568887170812033,386,418,1,805,48.
 Llama-3-Instruct-8B-WPO-HB-v2,57.33198613024009,1.4953200715726744,469,336,0,805,58.26086956521739,community,2472,53.37264268894168,0.7120573420060313
 claude-3-5-sonnet-20240620,40.56021409682828,1.4679655403720542,312,493,0,805,38.75776397515528,community,1488,52.36675427146999,
 yi-large-preview,57.46724251946292,1.4305696667082746,463,338,4,805,57.7639751552795,verified,2335,51.894415134099546,
+Llama-3-Instruct-8B-RainbowPO,47.91794368953007,1.4908245699081568,385,419,1,805,47.88819875776398,community,1878,51.66066005580552,0.7827000488843416
 gpt4_1106_preview_verbose,64.30360147101865,1.3348590089025316,525,268,12,805,65.96273291925466,dev,2402,51.57500797967598,
 gpt-4o-mini-2024-07-18,44.65413862507926,1.4572395578449813,350,451,4,805,43.72670807453416,minimal,1861,50.727144855901976,0.8284734951761676
 Storm-7B,50.26886905528583,1.4728176780737183,397,408,0,805,49.31677018633541,community,2045,50.45110959343775,

diff --git a/...ights/weighted_alpaca_eval_gpt4_turbo/length_controlled_v1/baseline_gpt4_1106_preview.csv b/...ights/weighted_alpaca_eval_gpt4_turbo/length_controlled_v1/baseline_gpt4_1106_preview.csv
@@ -192,3 +192,4 @@ SelfMoA_gemma-2-9b-it-WPO-HB,0.2523363342614252,0.3970191588440620,1.41373511384
 NullModel,-1.0518971527519405,0.2538445948493148,1.9057926500734572
 GPO-Llama-3-8B-Instruct-GPM-2B,-1.1688688988236986,0.7678817822697138,-0.4997466376902971
 SPPO-Llama-3-8B-Instruct-GPM-2B,-1.2289746990068291,0.8046474033904255,-0.6767509934260389
+Llama-3-Instruct-8B-RainbowPO,-1.3587935106099684,0.7600298380500641,0.1779421196386809
diff --git a/src/alpaca_eval/models_configs/Llama-3-Instruct-8B-RainbowPO/configs.yaml b/src/alpaca_eval/models_configs/Llama-3-Instruct-8B-RainbowPO/configs.yaml
@@ -0,0 +1,17 @@
+Llama-3-Instruct-8B-RainbowPO:
+  completions_kwargs:
+    batch_size: 900
+    use_beam_search: true
+    max_new_tokens: 4096
+    model_kwargs:
+      dtype: bfloat16
+    model_name: BraceZHY/Llama-3-8B-Instruct-RainbowPO
+    stop_token_ids:
+    - 128001
+    - 128009
+    temperature: 0.9
+    top_p: 1.0
+  fn_completions: vllm_local_completions
+  pretty_name: Llama-3-Instruct-8B-RainbowPO 
+  prompt_template: Llama-3-Instruct-8B-RainbowPO/prompt.txt
+  link: https://huggingface.co/BraceZHY/Llama-3-8B-Instruct-RainbowPO
diff --git a/src/alpaca_eval/models_configs/Llama-3-Instruct-8B-RainbowPO/prompt.txt b/src/alpaca_eval/models_configs/Llama-3-Instruct-8B-RainbowPO/prompt.txt
@@ -0,0 +1,5 @@
+<|begin_of_text|><|start_header_id|>user<|end_header_id|>
+
+{instruction}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,5 @@
		<\|begin_of_text\|><\|start_header_id\|>user<\|end_header_id\|>

		{instruction}<\|eot_id\|><\|start_header_id\|>assistant<\|end_header_id\|>