v2 wip

bulk-eval.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ "$1" == "" ]; then
-    GLOB="results/interview_junior*.ndjson"
+    GLOB="results-v2/interview_junior*.ndjson"
 else
     GLOB="$1/interview_junior*.ndjson"
 fi

compare.py

@@ -119,7 +119,7 @@ def analysis(data, analyser):
 
     return data
 
-def main(config: str, path: str = "results/", analyser: str = "", language: str = "javascript,python"):
+def main(config: str, path: str = "results-v2/", analyser: str = "", language: str = "javascript,python"):
     cfg = yaml.safe_load(open(config))
 
     for lang in language.split(','):

evaluate.py

@@ -52,7 +52,7 @@ def evaluation(test, language, code):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Interview evaluator')
-    parser.add_argument('--interview', type=str, default='junior-dev', help='interview to evaluate')
+    parser.add_argument('--interview', type=str, default='junior-v2', help='interview to evaluate')
     parser.add_argument('--input', type=str, required=True, help='path to interview*.ndjson')
     parser.add_argument('--test', type=str, help='(optional) specific test to evaluate')
     parser.add_argument('--stopcomment', action='store_true', help='(optional) stop code extraction at first comment')

junior-v2/gcd.yaml

@@ -11,6 +11,9 @@
     gcd_5_0:
       assert: "f.call(5,0)"
       eq: 5
+    gcd_0_3:
+      assert: "f.call(0,3)"
+      eq: 3
     gcd_13_7:
       assert: "f.call(13,7)"
       eq: 1

prepare.py

@@ -8,7 +8,7 @@
 from jinja2 import Template
 from pathlib import Path
 
-def load_questions(interview='junior-dev'):
+def load_questions(interview='junior-v2'):
     module_dir = os.path.dirname(os.path.abspath(__file__))
     for file_path in glob.glob(module_dir+'/'+interview+'/*.yaml'):
         with open(file_path, 'r') as file:
@@ -34,14 +34,14 @@ def save_interview(input, templateout, params, model, results):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Interview preparation')
     parser.add_argument('--language', type=str, default='python,javascript', help='languages to prepare, comma seperated')
-    parser.add_argument('--interview', type=str, default='junior-dev', help='interview to prepare')
+    parser.add_argument('--interview', type=str, default='junior-v2', help='interview to prepare')
     parser.add_argument('--template', type=str, required=True, help='prompt template file')
     args = parser.parse_args()
 
     template = Template(open(args.template).read())
     template_name = Path(args.template).stem
 
-    output_filename = f"results/prepare_{args.interview}_{args.language.replace(',', '-')}_{template_name}.ndjson"
+    output_filename = f"results-v2/prepare_{args.interview}_{args.language.replace(',', '-')}_{template_name}.ndjson"
     outputs = []
     for test in load_questions(interview=args.interview):
         for language in args.language.split(','):