Merge pull request #2 from SilenNaihin/benchmark-integration

add reports, consolidate, update benchmark files

benchmarks.py → agbenchmark/benchmarks.py

@@ -7,19 +7,20 @@
 
 def run_specific_agent(task: str) -> Tuple[str, int]:
     # Construct the command
-    command = ['python', 'main_no_modal.py', task]
+    command = ["python", "main_no_modal.py", task]
     subprocess.run(command, text=True)
 
+
 def execute_generated_files():
     # Navigate to generated directory
-    os.chdir('generated')
+    os.chdir("generated")
 
     # Iterate over every .txt file in the directory
-    for file_name in glob.glob('*.txt'):
-        with open(file_name, 'r') as file:
+    for file_name in glob.glob("../*.txt"):
+        with open(file_name, "r") as file:
             python_code = file.read()
-            python_code = python_code.replace('```python', '')
-            python_code = python_code.replace('```', '')
+            python_code = python_code.replace("```python", "")
+            python_code = python_code.replace("```", "")
             # Execute the code in the .txt file
             exec(python_code)
 

agbenchmark/config.json

@@ -0,0 +1,5 @@
+{
+  "workspace": "generated",
+  "entry_path": "agbenchmark/benchmarks.py",
+  "cutoff": 60
+}

regression_tests.json → agbenchmark/regression_tests.json

@@ -2,6 +2,7 @@
     "TestWriteFile": {
         "difficulty": "basic",
         "dependencies": [],
-        "test": "agbenchmark/tests/basic_abilities/write_file/write_file_test.py"
+        "test": "agbenchmark/challenges/interface/write_file",
+        "success": true
     }
 }

agbenchmark/reports/1.json

@@ -0,0 +1,18 @@
+{
+    "command": "agbenchmark start --test TestWriteFile --mock",
+    "completion_time": "2023-07-10-21:19",
+    "time_elapsed": "8.34 seconds",
+    "tests": {
+        "TestWriteFile": {
+            "difficulty": "basic",
+            "dependencies": [],
+            "test": "agbenchmark/challenges/interface/write_file",
+            "success": true
+        }
+    },
+    "config": {
+        "workspace": "generated",
+        "entry_path": "agbenchmark/benchmarks.py",
+        "cutoff": 60
+    }
+}