pytest integration

.github/workflows/python.yaml

@@ -6,10 +6,18 @@ on:
     branches: [ master ]
 jobs:
   build:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [pypy3, 3.5, 3.6, 3.7, 3.8, 3.9]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [pypy3, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10"]
+        exclude:
+          # apparently win/mac don't have pypy3
+          - os: windows-latest
+            python-version: pypy3
+          - os: macos-latest
+            python-version: pypy3
+    runs-on: ${{ matrix.os }}
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -19,7 +27,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install Pygments>=2.5.2
+          pip install Pygments>=2.5.2 pytest
       - name: Test
+        env:
+          PYTHONPATH: lib
         run: |
-          make testone
+          py.test -k "not knownfailure" test
+

pyproject.toml

@@ -0,0 +1,85 @@
+[tool.pytest.ini_options]
+# callecta ll tags with:
+# cat test/tm-cases/*tags \
+#  | sed 's/#.*//' \
+#  | tr ' ' '\n' \
+#  | sort \
+#  | uniq \
+#  | tr '-' '_' \
+#  | awk '{print "    \"" $1 "\","}'
+
+markers = [
+    "blank",
+    "code_color",
+    "code_as_com",
+    "code_friendly",
+    "cuddle_lists",
+    "dontcare",
+    "emacs",
+    "eol",
+    "escape",
+    "extra",
+    "extras",
+    "fenced_code_blocks",
+    "footnotes",
+    "fromphpmarkdown",
+    "header_ids",
+    "highlightjs_lang",
+    "html_classes",
+    "htmlentities",
+    "indentation",
+    "issue104",
+    "issue113",
+    "issue127",
+    "issue135",
+    "issue15",
+    "issue16",
+    "issue165",
+    "issue18",
+    "issue21",
+    "issue213",
+    "issue216",
+    "issue24",
+    "issue26",
+    "issue27",
+    "issue3",
+    "issue30",
+    "issue33",
+    "issue36",
+    "issue42",
+    "issue52",
+    "issue54",
+    "issue57",
+    "issue67",
+    "issue7",
+    "issue74",
+    "issue76",
+    "issue78",
+    "issue84",
+    "issue86",
+    "issue87",
+    "issue9",
+    "issue90",
+    "knownfailure",
+    "link_patterns",
+    "markdown_in_html",
+    "metadata",
+    "nofollow",
+    "pi",
+    "pygments",
+    "pyshell",
+    "questionable",
+    "safe_mode",
+    "smarty_pants",
+    "smedberg",
+    "strike",
+    "tables",
+    "tag_friendly",
+    "task_list",
+    "toc",
+    "underline",
+    "unicode",
+    "wiki_tables",
+    "xinclude",
+    "xml",
+]

test/README.md

@@ -1,3 +1,5 @@
+## Introduction
+
 This directory holds test suite. There are a number of test sets, each in its own directory:
 
 - **tm-cases**: Cases I wrote while writing markdown2.py. Many of these are
@@ -9,80 +11,59 @@ This directory holds test suite. There are a number of test sets, each in its ow
 - **php-markdown-extra-cases**: Test cases included in the MDTest package
   (same as above) testing extra Markdown syntax that only PHP Markdown implements.
 
+## markdown2.py test results
 
-# markdown2.py test results
-
-To run the test suite:
+To run all the tests:
 
-    python test.py [TAGS...]
+    pytest -vss test
+    (or within the test dir)
+    pytest -vss .
 
-The test driver used (testlib.py) allows one to filter the tests run via short
-strings that identify specific or groups of tests. Run `python test.py -l` to
+The test driver used (pytest) allows one to filter the tests run via short
+strings that identify specific or groups of tests. Run `pytest test --list` to
 list all available tests and their names/tags. I use the "knownfailure" tag to
 mark those tests that I know fail (e.g. the `php-markdown-extra-cases` all fail
 because markdown2.py doesn't implement those additions to the Markdown syntax).
 To run the test suite **without** the known failures:
 
-    $ python test.py -- -knownfailure
-    markdown2/tm/auto_link ... ok
-    markdown2/tm/blockquote ... ok
-    markdown2/tm/blockquote_with_pre ... ok
-    markdown2/tm/code_block_with_tabs [fromphpmarkdown] ... ok
-    markdown2/tm/code_safe_emphasis [code_safe] ... ok
-    markdown2/tm/codeblock ... ok
-    markdown2/tm/codespans ... ok
-    markdown2/tm/emphasis ... ok
-    markdown2/tm/escapes ... ok
-    markdown2/tm/header ... ok
-    markdown2/tm/hr ... ok
-    markdown2/tm/inline_links ... ok
-    markdown2/tm/lists ... ok
-    markdown2/tm/nested_list ... ok
-    markdown2/tm/parens_in_url_4 [fromphpmarkdown] ... ok
-    markdown2/tm/raw_html ... ok
-    markdown2/tm/ref_links ... ok
-    markdown2/tm/safe_mode ... ok
-    markdown2/tm/sublist-para [questionable] ... ok
-    markdown2/tm/tricky_anchors ... ok
-    markdown2/tm/underline_in_autolink ... ok
-    markdown2/markdowntest/amps_and_angle_encoding ... ok
-    markdown2/markdowntest/auto_links ... ok
-    markdown2/markdowntest/backslash_escapes ... ok
-    markdown2/markdowntest/blockquotes_with_code_blocks ... ok
-    markdown2/markdowntest/hard-wrapped_paragraphs_with_list-like_lines ... ok
-    markdown2/markdowntest/horizontal_rules ... ok
-    markdown2/markdowntest/inline_html_simple ... ok
-    markdown2/markdowntest/inline_html_comments ... ok
-    markdown2/markdowntest/links_inline_style ... ok
-    markdown2/markdowntest/links_reference_style ... ok
-    markdown2/markdowntest/literal_quotes_in_titles ... ok
-    markdown2/markdowntest/markdown_documentation_basics ... ok
-    markdown2/markdowntest/markdown_documentation_syntax ... ok
-    markdown2/markdowntest/nested_blockquotes ... ok
-    markdown2/markdowntest/ordered_and_unordered_lists ... ok
-    markdown2/markdowntest/strong_and_em_together ... ok
-    markdown2/markdowntest/tabs ... ok
-    markdown2/phpmarkdown/backslash_escapes ... ok
-    markdown2/phpmarkdown/code_spans ... ok
-    markdown2/phpmarkdown/email_auto_links ... ok
-    markdown2/phpmarkdown/headers ... ok
-    markdown2/phpmarkdown/images_untitled ... ok
-    markdown2/phpmarkdown/inline_html_comments ... ok
-    markdown2/phpmarkdown/ins_&_del ... ok
-    markdown2/phpmarkdown/links_inline_style ... ok
-    markdown2/phpmarkdown/md5_hashes ... ok
-    markdown2/phpmarkdown/php-specific_bugs ... ok
-    markdown2/phpmarkdown/tight_blocks ... ok
-    markdown2/direct/code_in_strong [code, strong] ... ok
-    markdown2/direct/pre ... ok
-    markdown2/direct/starter_pre [pre, recipes] ... ok
-
-    ----------------------------------------------------------------------
-    Ran 52 tests in 0.799s
-
-    OK
-
-
-TODO: Add details about which tests in the various test sets that markdown2.py
+    $ pytest test_rendering.py -m "not knownfailure
+    test/test_rendering.py::test_render[tm-cases-CVE-2018-5773.text] PASSED
+    test/test_rendering.py::test_render[tm-cases-ampersands.text] PASSED
+    test/test_rendering.py::test_render[tm-cases-auto_link.text] FAILED
+    test/test_rendering.py::test_render[tm-cases-auto_link_email_with_underscore.text] FAILED
+    ....
+    ....
+    5 failed, 146 passed, 15 deselected in 0.33s
+
+
+## Examples
+
+List all tests (and tags):
+
+    pytest test --list
+
+Run all tests:
+
+    pytest -vss test
+
+Run one single (named test):
+
+    pytest -s test/test_rendering.py::test_render[tm-cases-codespans.text]
+
+Run all tests with a particular flag:
+
+    pytest -vvs  test -m pygments
+
+RUn all test with flags matching an expression:
+
+    pytest -vvs  test -m "pygments and not fenced_code_blocks"
+
+
+**NOTE**: All the commands are executed from the top most check out directory, PYTHONPATH is set to lib.
+
+**NOTE**: pass the -vv flag to display verbose logging and -s flag to prevent pytest to capture the stdout/stderr (for debug)
+
+
+**TODO**: Add details about which tests in the various test sets that markdown2.py
 fails... and why I'm not concerned about them. 
 

test/conftest.py

@@ -0,0 +1,90 @@
+import re  # pylint: disable=unused-import
+import os
+import warnings
+from pathlib import Path
+
+import pytest
+
+ALL_SUBDIRS = []
+
+
+def _gather_md_tests(srcdir):
+    def sanitize(txt):
+        return txt.replace(" ", "").replace("-", "_").replace(".", "_")
+
+    class Item:
+        def __init__(self, src):
+            self.src = src
+            self.expected = src.with_suffix(".html")
+            assert self.expected.exists()
+
+        @property
+        def markers(self):
+            src = self.src.with_suffix(".tags")
+            if src.exists():
+                marks = []
+                with src.open() as fp:
+                    for line in fp:
+                        if not line.strip() or line.strip().startswith("#"):
+                            continue
+                        marks.extend(line.partition("#")[0].split())
+                return [sanitize(m) for m in marks]
+
+        @property
+        def options(self):
+            src = self.src.with_suffix(".opts")
+            if not src.exists():
+                return
+            with warnings.catch_warnings():
+                # files as link_patterns_double_hit.opts trigger "invalid escape sequence \s"
+                warnings.simplefilter("ignore")
+                return eval(src.read_text())
+
+    items = {}
+    for path in srcdir.glob("*.text"):
+        name = "{parent}-{name}".format(parent=srcdir.name, name=path.with_suffix('').name)
+        assert name not in items
+        items[name] = Item(path)
+    return items
+
+
+def parametrize(arguments, subdir):
+    global ALL_SUBDIRS
+    datadir = Path(os.getenv("DATADIR", Path(__file__).parent)).absolute()
+    srcdir = datadir / subdir
+
+    ALL_SUBDIRS.append(srcdir)
+    items = _gather_md_tests(srcdir)
+
+    # we build the parametrized
+    def _fn(fn):
+        parameters = []
+        for name, item in sorted(items.items()):
+            marks = [getattr(pytest.mark, m) for m in item.markers or []]
+            args = (item.src, item.expected, item.options)
+            if "marks" in arguments:
+                args = [*args, set(m.name for m in marks)]
+            kwargs = {"id": name, "marks": marks}
+            param = pytest.param(*args, **kwargs)
+            parameters.append(param)
+        return pytest.mark.parametrize(arguments, parameters)(fn)
+
+    return _fn
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("helloworld")
+    group.addoption(
+        "--list",
+        action="store_true",
+        dest="list-tests",
+        help="list all tests",
+    )
+
+
+def pytest_collection_finish(session):
+    if getattr(session.config.option, "list-tests"):
+        for subdir in ALL_SUBDIRS:
+            for name, item in sorted(_gather_md_tests(subdir).items()):
+                print("{} {}".format(name, item.markers))
+        pytest.exit("Done!")

test/php-markdown-cases/Email auto links.tags

@@ -0,0 +1 @@
+htmlentities unicode

test/php-markdown-extra-cases/Abbr.tags

@@ -1 +1 @@
-knownfailure
+knownfailure unicode