refactor!: compile each language individually

.editorconfig

@@ -13,5 +13,5 @@ indent_size = 3
 [*.md]
 indent_size = 2
 
-[*.yml]
+[*.{yml,json}]
 indent_size = 2

.github/CONTRIBUTING.md

@@ -2,9 +2,7 @@
 
 ## Adding a new language
 
-- Add the language repo to _README.rst_ (along with its license) and
- _tree_sitter_languages/repos/repos.txt_.
-- Add the language name to _tests/test_tree_sitter_languages.py_ (sorted).
-- Add `TS_LANGUAGE_INIT(name)` and `TS_LANGUAGE_METHOD(name),` to
- _tree_sitter_languages/languages.c_ (sorted).
+- Add the language data to _languages.json_.
+- Add the language repo and license to _README.rst_.
+- Add the language name to _tests/test_tree_sitter_languages.py_.
 - Submit a pull request.

.github/workflows/release.yml

@@ -12,10 +12,6 @@ jobs:
  steps:
  - name: Checkout repository
  uses: actions/checkout@v4
- - name: Clone repos
- working-directory: tree_sitter_languages/repos
- shell: bash
- run: ./clone.sh
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
@@ -41,10 +37,6 @@ jobs:
  steps:
  - name: Checkout repository
  uses: actions/checkout@v4
- - name: Clone repos
- working-directory: tree_sitter_languages/repos
- shell: bash
- run: ./clone.sh
  - name: Set up Python
  uses: actions/setup-python@v5
  with:

.github/workflows/tests.yml

@@ -4,10 +4,14 @@ on:
  push:
  branches: ["*"]
  paths:
+ - setup.py
+ - languages.json
  - tests/
  - tree_sitter_languages/
  pull_request:
  paths:
+ - setup.py
+ - languages.json
  - tests/
  - tree_sitter_languages/
 
@@ -19,16 +23,16 @@ jobs:
  steps:
  - name: Checkout repository
  uses: actions/checkout@v4
- - name: Clone repos
- working-directory: tree_sitter_languages/repos
- shell: bash
- run: ./clone.sh
  - name: Set up Python
  uses: actions/setup-python@v5
  with:
  cache: pip
  python-version: "3.9"
+ - name: Set up tree-sitter CLI
+ uses: tree-sitter/setup-action/cli/@v1
  - name: Install package
  run: pip install -v -e .
+ env:
+ TS_REGENERATE: "1"
  - name: Run tests
- run: python -munittest discover tests
+ run: python -munittest discover -v tests

MANIFEST.in

@@ -1,4 +1,3 @@
-include tree_sitter_languages/languages.h
+include languages.json
 
-prune tree_sitter_languages/repos
-recursive-include tree_sitter_languages/repos src/**/*.[ch]
+prune vendor

README.rst

@@ -17,6 +17,9 @@ Install
 
  pip install tree-sitter-languages
 
+
+**Note:** building from source requires ``git`` and a C compiler.
+
 Usage
 =====
 

languages.json

@@ -0,0 +1,50 @@
+{
+ "dot": {
+ "repo": "https://github.com/rydesun/tree-sitter-dot"
+ },
+ "elisp": {
+ "repo": "https://github.com/Wilfred/tree-sitter-elisp"
+ },
+ "elm": {
+ "repo": "https://github.com/elm-tooling/tree-sitter-elm"
+ },
+ "fixed_form_fortran": {
+ "repo": "https://github.com/ZedThree/tree-sitter-fixed-form-fortran",
+ "branch": "f77"
+ },
+ "fortran": {
+ "repo": "https://github.com/stadelmanma/tree-sitter-fortran"
+ },
+ "gomod": {
+ "repo": "https://github.com/camdencheek/tree-sitter-go-mod"
+ },
+ "hack": {
+ "repo": "https://github.com/slackhq/tree-sitter-hack"
+ },
+ "hcl": {
+ "repo": "https://github.com/tree-sitter-grammars/tree-sitter-hcl",
+ "languages": {
+ "hcl": "",
+ "terraform": "dialects/terraform"
+ }
+ },
+ "kotlin": {
+ "repo": "https://github.com/fwcd/tree-sitter-kotlin"
+ },
+ "make": {
+ "repo": "https://github.com/tree-sitter-grammars/tree-sitter-make"
+ },
+ "objc": {
+ "repo": "https://github.com/tree-sitter-grammars/tree-sitter-objc"
+ },
+ "rst": {
+ "repo": "https://github.com/stsewd/tree-sitter-rst"
+ },
+ "scala": {
+ "repo": "https://github.com/tree-sitter/tree-sitter-scala"
+ },
+ "sql": {
+ "repo": "https://github.com/derekstride/tree-sitter-sql",
+ "branch": "gh-pages"
+ }
+}

setup.py

@@ -1,8 +1,90 @@
-from glob import glob
+from json import loads
+from pathlib import Path
 from platform import system
+from os import chdir, environ, getcwd
+from typing import TypedDict
 
-from setuptools import Extension, setup # type: ignore
-from wheel.bdist_wheel import bdist_wheel # type: ignore
+from setuptools import Extension, setup
+from setuptools.command.build_ext import build_ext
+from wheel.bdist_wheel import bdist_wheel
+
+Language = TypedDict("Language", {
+ "repo": str,
+ "branch": str | None,
+ "languages": dict[str, str] | None
+})
+
+languages: dict[str, Language] = \
+ loads(Path(__file__).with_name("languages.json").read_text())
+
+extensions: list[Extension] = []
+
+common_source = Path(__file__).parent / "tree_sitter_languages" / "language.c"
+
+for lang, data in languages.items():
+ for name in data.get("languages", {lang: ""}).keys():
+ extensions.append(
+ Extension(
+ name=f"tree_sitter_languages._language.{name}",
+ sources=[str(common_source)],
+ include_dirs=[lang],
+ define_macros=[
+ ("PY_SSIZE_T_CLEAN", None),
+ ("TREE_SITTER_HIDE_SYMBOLS", None),
+ ("TS_LANGUAGE_NAME", name),
+ ],
+ extra_compile_args=[
+ "-std=c11",
+ "-fvisibility=hidden",
+ "-Wno-cast-function-type",
+ "-Wno-unused-but-set-variable",
+ "-Werror=implicit-function-declaration",
+ ] if system() != "Windows" else [
+ "/std:c11",
+ "/wd4244",
+ ],
+ py_limited_api=True,
+ optional=True,
+ )
+ )
+
+
+class BuildExt(build_ext):
+ def build_extension(self, ext: Extension):
+ name = ext.include_dirs.pop()
+ lang = languages[name]
+ cwd = getcwd()
+
+ if not (dir := Path(cwd) / "vendor" / name).is_dir():
+ clone = ["git", "clone", "-q", "--depth=1", "--sparse"]
+ if branch := lang.get("branch"):
+ clone.append(f"--branch={branch}")
+ clone.extend([lang["repo"], dir.relative_to(cwd)])
+ self.spawn(clone)
+ self.spawn([
+ "git", "-C", dir.relative_to(cwd),
+ "sparse-checkout", "set", "--no-cone", "/**/src/**"
+ ])
+ else:
+ self.spawn([
+ "git", "-C", dir.relative_to(cwd),
+ "pull", "-q", "--depth=1"
+ ])
+
+ name = ext.name.split(".")[-1]
+ path = dir / lang.get("languages", {name: ""})[name]
+ src = path / "src"
+ if "TS_REGENERATE" in environ:
+ chdir(path)
+ self.spawn([
+ "tree-sitter", "generate",
+ "--no-bindings", "src/grammar.json"
+ ])
+ chdir(cwd)
+ ext.sources.extend(list(map(str, src.glob("*.c"))))
+ ext.include_dirs = [str(src)]
+
+ return super().build_extension(ext)
 
 
 class BdistWheel(bdist_wheel):
@@ -13,32 +95,15 @@ def get_tag(self):
  return python, abi, platform
 
 
-sources = glob('tree_sitter_languages/repos/**/src/*.c', recursive=True)
-sources.append("tree_sitter_languages/languages.c")
-
 setup(
- packages=["tree_sitter_languages"],
- include_package_data=False,
- ext_modules=[
- Extension(
- name="tree_sitter_languages.languages",
- sources=sources,
- define_macros=[
- ("PY_SSIZE_T_CLEAN", None),
- ("TREE_SITTER_HIDE_SYMBOLS", None),
- ],
- extra_compile_args=[
- "-std=c11",
- "-fvisibility=hidden",
- "-Wno-cast-function-type",
- "-Wno-unused-but-set-variable",
- "-Werror=implicit-function-declaration",
- ] if system() != "Windows" else [
- "/std:c11",
- "/wd4244",
- ],
- py_limited_api=True
- )
+ packages=[
+ "tree_sitter_languages",
+ "tree_sitter_languages._language",
  ],
- cmdclass={"bdist_wheel": BdistWheel},
+ include_package_data=False,
+ ext_modules=extensions,
+ cmdclass={
+ "build_ext": BuildExt,
+ "bdist_wheel": BdistWheel,
+ },
 )

tests/test_tree_sitter_languages.py

@@ -34,5 +34,5 @@ def test_get_language(self):
  self.assertIsInstance(get_language(language), Language)
 
  def test_invalid_name(self):
- self.assertRaises(AttributeError, get_language, "invalid")
- self.assertRaises(AttributeError, get_parser, "invalid")
+ self.assertRaises(LookupError, get_language, "invalid")
+ self.assertRaises(LookupError, get_parser, "invalid")

tree_sitter_languages/__init__.py

@@ -1,15 +1,18 @@
 """Tree-sitter languages"""
 
-from tree_sitter import Language, Parser
+from importlib import import_module as _import
 
-from . import languages
+from tree_sitter import Language, Parser
 
 
 def get_language(name: str) -> Language:
  """Get the language with the given name."""
- if not hasattr(languages, name):
- raise AttributeError(f"Language not found: {name}")
- return Language(getattr(languages, name)())
+ try:
+ module = _import(f"._language.{name}", __package__)
+ except ModuleNotFoundError:
+ raise LookupError(f"Language not found: {name}")
+ else:
+ return Language(module.language())
 
 
 def get_parser(language: str) -> Parser:

tree_sitter_languages/_language/__init__.py

@@ -0,0 +1 @@
+

tree_sitter_languages/language.c

@@ -0,0 +1,39 @@
+#include <Python.h>
+
+typedef struct TSLanguage TSLanguage;
+
+#ifndef TS_LANGUAGE_NAME
+#error TS_LANGUAGE_NAME must be defined
+#endif
+
+#define _str(s) #s
+#define str(s) _str(s)
+#define _cat(a, b) a##b
+#define cat(a, b) _cat(a, b)
+
+#define TS_LANGUAGE_FUNC cat(tree_sitter_, TS_LANGUAGE_NAME)
+#define TS_LANGUAGE_METHOD cat(TS_LANGUAGE_NAME, _language)
+#define TS_LANGUAGE_MODULE cat(PyInit_, TS_LANGUAGE_NAME)
+
+TSLanguage *TS_LANGUAGE_FUNC(void);
+
+static PyObject* TS_LANGUAGE_METHOD(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) {
+ return PyLong_FromVoidPtr(TS_LANGUAGE_FUNC());
+}
+
+static PyMethodDef methods[] = {
+ {"language", TS_LANGUAGE_METHOD, METH_NOARGS, NULL},
+ {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef module = {
+ .m_base = PyModuleDef_HEAD_INIT,
+ .m_name = str(TS_LANGUAGE_NAME),
+ .m_doc = NULL,
+ .m_size = -1,
+ .m_methods = methods
+};
+
+PyMODINIT_FUNC TS_LANGUAGE_MODULE(void) {
+ return PyModule_Create(&module);
+}