latest

Author: abacaj

README.md

@@ -39,13 +39,12 @@ python eval_wizard.py
 
 Process the jsonl file to extract code samples from model completions
 
-**Note**: the replit base model does not go through this process
+**Note**: the replit base + instruct model does not go through this process
 
 ```sh
 # replace args for various models:
 # --path results/wizard --out_path results/wizard/eval.jsonl
 # --path results/opencode --out_path results/opencode/eval.jsonl
-# --path results/replit_instruct --out_path results/replit_instruct/eval.jsonl
 
 python process_eval.py --path results/wizard --out_path results/wizard/processed.jsonl --add_prompt
 ```
@@ -56,7 +55,7 @@ Then get the results
 # replace args for various models:
 # results/wizard/processed.jsonl
 # results/opencode/processed.jsonl
-# results/replit_instruct/processed.jsonl
+# results/replit_instruct/eval.jsonl
 # results/replit/eval.jsonl
 
 evaluate_functional_correctness results/wizard/processed.jsonl

core/evaluation.py

@@ -4,6 +4,7 @@
     PreTrainedTokenizer,
 )
 from tqdm import tqdm
+import itertools
 import typing
 
 BatchGenerator = typing.Callable[
@@ -17,13 +18,19 @@ def run_eval(
     num_samples_per_task: int,
     out_path: str,
     generate_batch_completion: BatchGenerator,
+    is_starcoder: bool = False,
 ):
     problems = read_problems()
+    # problems = dict(itertools.islice(problems.items(), 20))
     samples = []
     pbar = tqdm(total=len(problems) * num_samples_per_task)
 
     for task_id in problems:
-        prompt = problems[task_id]["prompt"].replace("    ", "\t")
+        if is_starcoder:
+            prompt = problems[task_id]["prompt"].replace("    ", "\t")
+        else:
+            prompt = problems[task_id]["prompt"]
+
         batch_completions = generate_batch_completion(
             model, tokenizer, prompt, num_samples_per_task
         )

eval_opencode.py

@@ -54,7 +54,9 @@ def tokenize_opencode(tokenizer: PreTrainedTokenizer, prompt: str):
 
     # verbose, but follows what is shown in the readme
     user = tokenizer("User:")
-    prompt_text = tokenizer(f"""Create a Python script for this problem: {prompt}""")
+    prompt_text = tokenizer(
+        f"""Complete the following Python code without any tests or explanation\n{prompt}"""
+    )
     eot_token = tokenizer("<|end_of_turn|>")
     assistant = tokenizer("Assistant:")
 
@@ -111,5 +113,10 @@ def convert_to_tensors(opencode_tokens: list[dict], device: torch.device):
     )
 
     run_eval(
-        model, tokenizer, num_samples_per_task, out_path, generate_batch_completion
+        model,
+        tokenizer,
+        num_samples_per_task,
+        out_path,
+        generate_batch_completion,
+        True,
     )

eval_replit.py

@@ -13,27 +13,12 @@
 TOKEN = ""
 
 
-# references: https://github.com/declare-lab/instruct-eval
-def count_indent(text: str) -> int:
-    count = 0
-    for char in text:
-        if char == " ":
-            count += 1
-        else:
-            break
-    return count
-
-
-def fix_indents(text: str, multiple: int = 2) -> str:
-    outputs = []
-    for line in text.split("\n"):
-        while count_indent(line) % multiple != 0:
-            line = " " + line
-        outputs.append(line)
-    return "\n".join(outputs)
+def fix_indents(text: str) -> str:
+    return text.replace("\t", "    ")
 
 
 def filter_code(completion: str) -> str:
+    # The program tends to overwrite, we only take the first function
     completion = completion.lstrip("\n")
     return completion.split("\n\n")[0]
 
@@ -42,7 +27,7 @@ def filter_code(completion: str) -> str:
 def generate_batch_completion(
     model: PreTrainedModel, tokenizer: PreTrainedTokenizer, prompt, batch_size
 ) -> list[str]:
-    prompt_input = f"""Complete the following Python code without any additional tests or explanations\n{prompt}"""
+    prompt_input = f"""Complete the following Python code without any tests or explanation\n{prompt}"""
 
     input_batch = [prompt_input for _ in range(batch_size)]
     inputs = tokenizer(input_batch, return_tensors="pt").to(model.device)
@@ -59,13 +44,13 @@ def generate_batch_completion(
         pad_token_id=tokenizer.pad_token_id,
     )
 
-    output = tokenizer.batch_decode(
+    batch_completions = tokenizer.batch_decode(
         [ids[input_ids_cutoff:] for ids in generated_ids],
         skip_special_tokens=True,
         clean_up_tokenization_spaces=False,
     )
 
-    return [filter_code(fix_indents(sample)) for sample in output]
+    return [filter_code(fix_indents(completion)) for completion in batch_completions]
 
 
 if __name__ == "__main__":
@@ -91,5 +76,9 @@ def generate_batch_completion(
     )
 
     run_eval(
-        model, tokenizer, num_samples_per_task, out_path, generate_batch_completion
+        model,
+        tokenizer,
+        num_samples_per_task,
+        out_path,
+        generate_batch_completion,
     )

eval_replit_instruct.py

@@ -13,25 +13,11 @@
 TOKEN = ""
 
 
-# references: https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder
-def format_output(output: str):
-    try:
-        return output.replace("\t", "    ")
-    except:
-        return ""
-
-
 @torch.inference_mode()
 def generate_batch_completion(
     model: PreTrainedModel, tokenizer: PreTrainedTokenizer, prompt: str, batch_size: int
 ) -> list[str]:
-    prompt_input = f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
-
-### Instruction:
-Create a Python script for this problem:
-{prompt}
-
-### Response:"""
+    prompt_input = f"""### Instruction:\nComplete the following Python code without any tests or explanation\n{prompt}\n\n### Response:"""
 
     input_batch = [prompt_input for _ in range(batch_size)]
     inputs = tokenizer(input_batch, return_tensors="pt").to(model.device)
@@ -54,7 +40,7 @@ def generate_batch_completion(
         clean_up_tokenization_spaces=False,
     )
 
-    return [format_output(out) for out in output]
+    return output
 
 
 if __name__ == "__main__":

eval_wizard.py

@@ -13,25 +13,11 @@
 TOKEN = ""
 
 
-# references: https://github.com/nlpxucan/WizardLM/tree/main/WizardCoder
-def format_output(output: str):
-    try:
-        return output.replace("\t", "    ")
-    except:
-        return ""
-
-
 @torch.inference_mode()
 def generate_batch_completion(
     model: PreTrainedModel, tokenizer: PreTrainedTokenizer, prompt: str, batch_size: int
 ) -> list[str]:
-    prompt_input = f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
-
-### Instruction:
-Create a Python script for this problem:
-{prompt}
-
-### Response:"""
+    prompt_input = f"""### Instruction:\nComplete the following Python code without any tests or explanation\n{prompt}\n\n### Response:"""
 
     input_batch = [prompt_input for _ in range(batch_size)]
     inputs = tokenizer(input_batch, return_tensors="pt").to(model.device)
@@ -53,7 +39,7 @@ def generate_batch_completion(
         skip_special_tokens=True,
     )
 
-    return [format_output(out) for out in output]
+    return output
 
 
 if __name__ == "__main__":
@@ -81,5 +67,10 @@ def generate_batch_completion(
     )
 
     run_eval(
-        model, tokenizer, num_samples_per_task, out_path, generate_batch_completion
+        model,
+        tokenizer,
+        num_samples_per_task,
+        out_path,
+        generate_batch_completion,
+        True,
     )

human-eval/human_eval/evaluation.py

@@ -13,7 +13,7 @@
 def estimate_pass_at_k(
     num_samples: Union[int, List[int], np.ndarray],
     num_correct: Union[List[int], np.ndarray],
-    k: int
+    k: int,
 ) -> np.ndarray:
     """
     Estimates pass@k of each problem and returns them in an array.
@@ -33,7 +33,9 @@ def estimator(n: int, c: int, k: int) -> float:
         assert len(num_samples) == len(num_correct)
         num_samples_it = iter(num_samples)
 
-    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+    )
 
 
 def evaluate_functional_correctness(
@@ -52,7 +54,6 @@ def evaluate_functional_correctness(
 
     # Check the generated samples against test suites.
     with ThreadPoolExecutor(max_workers=n_workers) as executor:
-
         futures = []
         completion_id = Counter()
         n_samples = 0
@@ -86,8 +87,11 @@ def evaluate_functional_correctness(
     correct = np.array(correct)
 
     ks = k
-    pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
-                 for k in ks if (total >= k).all()}
+    pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+        for k in ks
+        if (total >= k).all()
+    }
 
     # Finally, save the results in one file:
     def combine_results():