From 5fa714c9caf1a88af3caa2a636e2e7a062df9cfa Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 8 Aug 2025 14:18:41 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=94=A7=20Improve=20spec=20update=20sc?= =?UTF-8?q?ript?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/benchmark.yml | 2 +- .github/workflows/tests.yml | 3 + docs/contributing.md | 16 +++++ tests/test_cmark_spec/get_cmark_spec.py | 77 +++++++++++++++++++++++++ tests/test_cmark_spec/spec.sh | 26 --------- 5 files changed, 97 insertions(+), 27 deletions(-) create mode 100644 tests/test_cmark_spec/get_cmark_spec.py delete mode 100755 tests/test_cmark_spec/spec.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 9c96c035..37cb7e24 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: "3.10" - name: install pandoc uses: r-lib/actions/setup-pandoc@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b0b42b9..aebd2a7c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,6 +45,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -e .[testing,linkify] + - name: Check spec file is up to date + run: | + python tests/test_cmark_spec/get_cmark_spec.py - name: Run pytest run: | pytest tests/ --cov=markdown_it --cov-report=xml --cov-report=term-missing diff --git a/docs/contributing.md b/docs/contributing.md index eb73ccda..3a6d6aeb 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -61,6 +61,22 @@ For documentation build tests: >> make html-strict ``` +### Updating the commonmark specification + +If you need to update the commonmark specification, you can do so by running: + +```shell +>> cd markdown-it-py +>> python tests/test_cmark_spec/get_cmark_spec.py +``` + +or + +```shell +>> cd markdown-it-py +>> uv run tests/test_cmark_spec/get_cmark_spec.py +``` + ## Contributing a plugin 1. Does it already exist as JavaScript implementation ([see npm](https://www.npmjs.com/search?q=keywords:markdown-it-plugin))? diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py new file mode 100644 index 00000000..851cad75 --- /dev/null +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -0,0 +1,77 @@ +# /// script +# dependencies = [ +# "requests", +# ] +# /// +from pathlib import Path + +default_version = "0.30" +default_output_path = Path(__file__).parent / "commonmark.json" +default_fixture_test_path = ( + Path(__file__).parent.parent / "test_port" / "fixtures" / "commonmark_spec.md" +) + + +def create_argparser(): + import argparse + + parser = argparse.ArgumentParser(description="Download CommonMark spec JSON") + parser.add_argument( + "version", + nargs="?", + default=default_version, + help=f"CommonMark spec version to download (default: {default_version})", + ) + parser.add_argument( + "--output", + "-o", + type=Path, + default=default_output_path, + help=f"Output file path (default: {default_output_path})", + ) + parser.add_argument( + "--test-fixture", + type=Path, + default=default_fixture_test_path, + help=f"Write to test fixture (default: {default_fixture_test_path})", + ) + return parser + + +if __name__ == "__main__": + import requests # type: ignore[import-untyped] + + args = create_argparser().parse_args() + version: str = args.version + output_path: Path = args.output + write_to_test_fixture = True + test_fixture: Path = args.test_fixture + changed = False + url = f"https://spec.commonmark.org/{version}/spec.json" + print(f"Downloading CommonMark spec from {url}") + response = requests.get(url) + response.raise_for_status() + if not output_path.exists() or output_path.read_text() != response.text: + changed = True + with output_path.open("w") as f: + f.write(response.text) + print(f"Updated to {output_path}") + else: + print(f"Spec file {output_path} is up to date, not overwriting") + + if write_to_test_fixture: + data = response.json() + text = "" + for item in data: + text += "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + text += f"src line: {item['start_line'] - 1}\n\n" + text += f".\n{item['markdown']}.\n{item['html']}.\n\n" + if not test_fixture.exists() or test_fixture.read_text() != text: + changed = True + with test_fixture.open("w") as f: + f.write(text) + print(f"Also updated to {test_fixture}") + else: + print(f"Fixture file {test_fixture} is up to date, not overwriting") + + raise SystemExit(0 if not changed else 1) diff --git a/tests/test_cmark_spec/spec.sh b/tests/test_cmark_spec/spec.sh deleted file mode 100755 index c8513903..00000000 --- a/tests/test_cmark_spec/spec.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -REPO="https://github.com/commonmark/CommonMark.git" -VERSION="0.30" - -function main { - echo "Cloning from repo: $REPO..." - git clone --quiet $REPO - - echo "Using version $VERSION..." - cd "CommonMark" - git checkout --quiet $VERSION - - echo "Dumping tests file..." - python3 "test/spec_tests.py" --dump-tests > "../commonmark.json" - - echo "Cleaning up..." - cd .. - rm -rf CommonMark - - echo "Done." -} - -main From 80f7827c577ab8fd11011b7fd36354c08269f96f Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 8 Aug 2025 14:20:34 +0200 Subject: [PATCH 2/4] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 56bd9df2..86353250 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ testing = [ "pytest", "pytest-cov", "pytest-regressions", + "requests", ] benchmarking = [ "psutil", From 1311a1d8000b77d21f7de8fd1fd602771fa00b68 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 8 Aug 2025 14:23:03 +0200 Subject: [PATCH 3/4] check failure --- tests/test_cmark_spec/get_cmark_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py index 851cad75..8be5d05a 100644 --- a/tests/test_cmark_spec/get_cmark_spec.py +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -5,7 +5,7 @@ # /// from pathlib import Path -default_version = "0.30" +default_version = "0.31.2" default_output_path = Path(__file__).parent / "commonmark.json" default_fixture_test_path = ( Path(__file__).parent.parent / "test_port" / "fixtures" / "commonmark_spec.md" From 988d5425945d04048ecd16c4081b72a70326924e Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Fri, 8 Aug 2025 14:23:42 +0200 Subject: [PATCH 4/4] Update get_cmark_spec.py --- tests/test_cmark_spec/get_cmark_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cmark_spec/get_cmark_spec.py b/tests/test_cmark_spec/get_cmark_spec.py index 8be5d05a..851cad75 100644 --- a/tests/test_cmark_spec/get_cmark_spec.py +++ b/tests/test_cmark_spec/get_cmark_spec.py @@ -5,7 +5,7 @@ # /// from pathlib import Path -default_version = "0.31.2" +default_version = "0.30" default_output_path = Path(__file__).parent / "commonmark.json" default_fixture_test_path = ( Path(__file__).parent.parent / "test_port" / "fixtures" / "commonmark_spec.md"