Merge pull request #65 from princeton-nlp/pypi

Add PyPI Library Build Logic to Main Repo

.gitignore

@@ -160,15 +160,17 @@ cython_debug/
 #.idea/
 
 # Custom
-notebooks/
-*.patch
-data/repos/copies
+.api_key
 .keys
 .vscode/
 *.jsonl
 *.jsonl.*
+*.patch
+*.DS_Store
 analysis/**/*.json
 analysis/**/scratch*
 analysis/benchmark/plots/
 analysis/evaluation/*.csv
 analysis/evaluation/*.pdf
+data/repos/copies
+notebooks/

build_deploy.sh

@@ -0,0 +1,6 @@
+# !bin/bash
+
+python3 -m build
+
+python3 -m twine upload --skip-existing --repository pypi dist/*
+# python3 -m twine upload --skip-existing --repository testpypi dist/*

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ['setuptools>=42']
+build-backend = 'setuptools.build_meta'

setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+version = attr: swebench.__version__
+license_files = LICENSE

setup.py

@@ -0,0 +1,41 @@
+import setuptools
+
+with open('README.md', 'r', encoding='utf-8') as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name='swebench',
+    author='John Yang',
+    author_email='byjohnyang@gmail.com',
+    description='The official SWE-bench package - a benchmark for evaluating LMs on software engineering',
+    keywords='nlp, benchmark, code',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    url='https://swebench.com',
+    project_urls={
+        'Documentation': 'https://github.com/princeton-nlp/SWE-bench',
+        'Bug Reports': 'http://github.com/princeton-nlp/SWE-bench/issues',
+        'Source Code': 'http://github.com/princeton-nlp/SWE-bench',
+        'Website': 'https://swebench.com',
+    },
+    packages=setuptools.find_packages(),
+    classifiers=[
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3 :: Only',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: OS Independent',
+    ],
+    python_requires='>=3.8',
+    install_requires=[
+        'beautifulsoup4',
+        'chardet',
+        'datasets',
+        'ghapi',
+        'GitPython',
+        'python-dotenv',
+        'requests',
+        'rich',
+    ],
+    include_package_data=True,
+)

swebench/__init__.py

@@ -0,0 +1,76 @@
+__version__ = "1.0.1"
+
+from swebench.collect.build_dataset import main as build_dataset
+from swebench.collect.get_tasks_pipeline import main as get_tasks_pipeline
+from swebench.collect.print_pulls import main as print_pulls
+
+from swebench.harness.constants import (
+    KEY_INSTANCE_ID,
+    KEY_MODEL,
+    KEY_PREDICTION,
+    MAP_REPO_TO_TEST_FRAMEWORK,
+    MAP_VERSION_TO_INSTALL,
+)
+
+from swebench.harness.run_evaluation import (
+  main as run_evaluation,
+)
+
+from swebench.harness.utils import (
+    get_environment_yml,
+    get_instances,
+    get_requirements,
+)
+
+from swebench.metrics.conversion import (
+    convert_log_to_ground_truth
+)
+
+from swebench.metrics.getters import (
+    get_diffs,
+    get_logs_eval,
+    get_logs_gold,
+)
+
+from swebench.metrics.log_parsers import (
+    MAP_REPO_TO_PARSER,
+)
+
+from swebench.metrics.metrics import (
+    compute_fail_to_pass,
+    compute_fail_to_pass_unweighted,
+    compute_fail_to_pass_weighted,
+    compute_pass_to_pass,
+    compute_pass_to_pass_unweighted,
+    compute_pass_to_pass_weighted,
+    get_resolution_status,
+)
+
+from swebench.metrics.monitor import (
+    monitor_validation,
+    monitor_logs_same_diff,
+)
+
+from swebench.metrics.report import (
+    get_eval_report,
+    get_eval_reports_for_logs,
+    get_eval_reports_for_dir,
+    get_model_eval_summary,
+    get_model_report,
+)
+
+from swebench.versioning.constants import (
+    MAP_REPO_TO_VERSION_PATHS,
+    MAP_REPO_TO_VERSION_PATTERNS,
+)
+
+from swebench.versioning.get_versions import (
+    get_version,
+    map_version_to_task_instances,
+    get_versions_from_build,
+    get_versions_from_web,
+)
+
+from swebench.versioning.utils import (
+    split_instances,
+)

collect/build_dataset.py → swebench/collect/build_dataset.py

@@ -6,8 +6,7 @@
 import os
 from typing import Optional
 
-
-from utils import Repo, extract_patches, extract_problem_statement_and_hints
+from swebench.collect.utils import Repo, extract_patches, extract_problem_statement_and_hints
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"

collect/get_tasks_pipeline.py → swebench/collect/get_tasks_pipeline.py

@@ -6,9 +6,9 @@
 import traceback
 
 from dotenv import load_dotenv
-from build_dataset import main as build_dataset
-from print_pulls import main as print_pulls
 from multiprocessing import Pool
+from swebench.collect.build_dataset import main as build_dataset
+from swebench.collect.print_pulls import main as print_pulls
 
 
 load_dotenv()

collect/print_pulls.py → swebench/collect/print_pulls.py

@@ -9,7 +9,7 @@
 from typing import Optional
 
 from fastcore.xtras import obj2dict
-from utils import Repo
+from swebench.collect.utils import Repo
 
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"

harness/constants.py → swebench/harness/constants.py

@@ -3,6 +3,9 @@
         "python": "3.6",
         "packages": "numpy scipy cython pytest pandas matplotlib",
         "install": "pip install -v --no-use-pep517 --no-build-isolation -e .",
+        "arch_specific_packages": {
+            "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+        }
     }
     for k in ["0.20", "0.21", "0.22"]
 }
@@ -12,6 +15,9 @@
             "python": "3.7",
             "packages": "numpy scipy cython pytest pandas matplotlib",
             "install": "pip install -v --no-use-pep517 --no-build-isolation -e .",
+            "arch_specific_packages": {
+                "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            }
         }
         for k in ["0.23", "0.24"]
     }
@@ -22,6 +28,9 @@
             "python": "3.9",
             "packages": "numpy scipy cython pytest pandas matplotlib joblib threadpoolctl",
             "install": "pip install -v --no-use-pep517 --no-build-isolation -e .",
+            "arch_specific_packages": {
+                "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            }
         }
         for k in ["1.0", "1.1", "1.2", "1.3", "1.4"]
     }
@@ -50,11 +59,13 @@
         "python": "3.9",
         "packages": "requirements.txt",
         "install": "pip install -e .",
+        "pip_packages": "Werkzeug==2.2.2",
     },
     "2.1": {
         "python": "3.10",
         "packages": "requirements.txt",
         "install": "pip install -e .",
+        "pip_packages": "Werkzeug==2.2.2",
     },
 }
 MAP_VERSION_TO_INSTALL_FLASK.update(
@@ -74,6 +85,7 @@
             "python": "3.11",
             "packages": "requirements.txt",
             "install": "pip install -e .",
+            "pip_packages": "Werkzeug==2.2.2"
         }
         for k in ["2.2", "2.3"]
     }
@@ -162,9 +174,27 @@
 
 MAP_VERSION_TO_INSTALL_MATPLOTLIB = {
     k: {
-        "python": "3.9",
+        "python": "3.11",
         "packages": "environment.yml",
         "install": "python -m pip install -e .",
+        "pip_packages": " ".join([
+            "contourpy==1.1.0",
+            "cycler==0.11.0",
+            "fonttools==4.42.1",
+            "kiwisolver==1.4.5",
+            "numpy==1.25.2",
+            "packaging==23.1",
+            "pillow==10.0.0",
+            "pyparsing==3.0.9",
+            "python-dateutil==2.8.2",
+            "six==1.16.0",
+            "setuptools==68.1.2",
+            "setuptools-scm==7.1.0",
+            "typing-extensions==4.7.1",
+        ]),
+        "arch_specific_packages": {
+            "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+        }
     }
     for k in ["3.5", "3.6", "3.7"]
 }
@@ -174,6 +204,9 @@
             "python": "3.8",
             "packages": "requirements.txt",
             "install": "python -m pip install -e .",
+            "arch_specific_packages": {
+                "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            }
         }
         for k in ["3.1", "3.2", "3.3", "3.4"]
     }
@@ -184,6 +217,9 @@
             "python": "3.7",
             "packages": "requirements.txt",
             "install": "python -m pip install -e .",
+            "arch_specific_packages": {
+                "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            }
         }
         for k in ["3.0"]
     }
@@ -193,6 +229,9 @@
         k: {
             "python": "3.5",
             "install": "python setup.py build; python setup.py install",
+            "arch_specific_packages": {
+                "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            }
         }
         for k in ["2.0", "2.1", "2.2", "1.0", "1.1", "1.2", "1.3", "1.4", "1.5"]
     }
@@ -204,15 +243,50 @@
         "pip_packages": "tox",
         "install": "pip install -e .[test]",
         "pre_install": ["sed -i 's/pytest/pytest -rA/' tox.ini"],
+        "arch_specific_packages": {
+            "aarch64": "gxx_linux-aarch64 gcc_linux-aarch64 make",
+            "x86_64": "gxx_linux-64 gcc_linux-64 make",
+        }
     } for k in
         ["1.5", "1.6", "1.7", "1.8", "2.0", "2.1", "2.2", "2.3", "2.4", "3.0"] + \
         ["3.1", "3.2", "3.3", "3.4", "3.5", "4.0", "4.1", "4.2", "4.3", "4.4"] + \
         ["4.5", "5.0", "5.1", "5.2", "5.3", "6.0", "6.2", "7.0", "7.1", "7.2"]
 }
-for k in ["3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "4.0"]:
+for k in ["3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "4.0", "4.1", "4.2", "4.3", "4.4"]:
     MAP_VERSION_TO_INSTALL_SPHINX[k][
         "pre_install"
-    ].append("sed -i 's/Jinja2>=2.3/Jinja2<3.1/' setup.py")
+    ].extend([
+        "sed -i 's/Jinja2>=2.3/Jinja2<3.0/' setup.py",
+        "sed -i 's/sphinxcontrib-applehelp/sphinxcontrib-applehelp<=1.0.7/' setup.py",
+        "sed -i 's/sphinxcontrib-devhelp/sphinxcontrib-devhelp<=1.0.5/' setup.py",
+        "sed -i 's/sphinxcontrib-qthelp/sphinxcontrib-qthelp<=1.0.6/' setup.py",
+        "sed -i 's/alabaster>=0.7,<0.8/alabaster>=0.7,<0.7.12/' setup.py",
+        'sed -i "s/\'packaging\',/\'packaging\', \'markupsafe<=2.0.1\',/" setup.py',
+    ])
+    if k in ["4.2", "4.3", "4.4"]:
+        MAP_VERSION_TO_INSTALL_SPHINX[k]["pre_install"].extend([
+            "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py",
+            "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py",
+        ])
+    elif k == "4.1":
+        MAP_VERSION_TO_INSTALL_SPHINX[k]["pre_install"].extend([
+            (
+                "grep -q 'sphinxcontrib-htmlhelp>=2.0.0' setup.py && "
+                "sed -i 's/sphinxcontrib-htmlhelp>=2.0.0/sphinxcontrib-htmlhelp>=2.0.0,<=2.0.4/' setup.py || "
+                "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py"
+            ),
+            (
+                "grep -q 'sphinxcontrib-serializinghtml>=1.1.5' setup.py && "
+                "sed -i 's/sphinxcontrib-serializinghtml>=1.1.5/sphinxcontrib-serializinghtml>=1.1.5,<=1.1.9/' setup.py || "
+                "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py"
+            )
+        ])
+    else:
+        MAP_VERSION_TO_INSTALL_SPHINX[k]["pre_install"].extend([
+            "sed -i 's/sphinxcontrib-htmlhelp/sphinxcontrib-htmlhelp<=2.0.4/' setup.py",
+            "sed -i 's/sphinxcontrib-serializinghtml/sphinxcontrib-serializinghtml<=1.1.9/' setup.py",
+        ])
+
 
 MAP_VERSION_TO_INSTALL_ASTROPY = {
     k: {"python": "3.9", "install": "pip install -e .[test]"}
@@ -247,6 +321,10 @@
     k: {"python": "3.9", "packages": "requirements.txt", "install": "pip install -e ."}
     for k in ["2.10", "2.11", "2.13", "2.14", "2.15", "2.16", "2.17", "2.8", "2.9", "3.0"]
 }
+MAP_VERSION_TO_INSTALL_PYLINT.update({
+    k: {**MAP_VERSION_TO_INSTALL_PYLINT[k], "pip_packages": " ".join([
+        "astroid==3.0.0a7"
+    ])} for k in ['3.0']})
 
 MAP_VERSION_TO_INSTALL_XARRAY = {
     k: {
@@ -472,4 +550,24 @@
 
 # Constants - Miscellaneous
 NON_TEST_EXTS = [".json", ".png", "csv", ".txt", ".md", ".jpg", ".jpeg", ".pkl", ".yml", ".yaml", ".toml"]
-SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/"
+SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/"
+
+# Constants - Repo/Version Mapped to Appropriate Conda Link
+MAP_REPO_VERSION_TO_CONDA_LINK = {
+    "django/django": {
+        "1.11": "py311_23.9.0-0",
+    },
+    "matplotlib/matplotlib": {
+        "3.1": "py311_23.9.0-0",
+        "3.2": "py311_23.9.0-0",
+        "3.3": "py311_23.9.0-0",
+        "3.4": "py311_23.9.0-0",
+        "3.0": "py311_23.10.0-1",
+    },
+    "mwaskom/seaborn": {"0.11": None, "0.12": None, "0.13": None},
+    "sympy/sympy": {
+        "1.0": "py39_23.9.0-0",
+    },
+}
+
+DEFAULT_CONDA_LINK = "py39_23.10.0-1"