From 2cdca2ea47ab428087cd9597748a0f87391f60ae Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 18 Nov 2023 16:17:30 -0800 Subject: [PATCH] MAINT Make doctests run; run some doctests in pyodide (#4280) This uses pyodide/pytest-pyodide#117 to run doctests in Pyodide. I also turned on and fixed various doctests that were not working for unrelated reasons --- conftest.py | 9 -- .../sphinx_pyodide/mdn_xrefs.py | 1 + src/py/_pyodide/_base.py | 21 ++-- src/py/_pyodide/_core_docs.py | 96 ++++++++++++------- src/py/pyodide/http.py | 23 ++--- 5 files changed, 87 insertions(+), 63 deletions(-) diff --git a/conftest.py b/conftest.py index e76c347e6a9..136a0bbc85a 100644 --- a/conftest.py +++ b/conftest.py @@ -154,16 +154,7 @@ def pytest_collection_modifyitems(config, items): cache = config.cache prev_test_result = cache.get("cache/lasttestresult", {}) - skipped_docstrings = [ - "_pyodide._base.CodeRunner", - "pyodide.http.open_url", - "pyodide.http.pyfetch", - ] - for item in items: - if isinstance(item, pytest.DoctestItem) and item.name in skipped_docstrings: - item.add_marker(pytest.mark.skip(reason="skipped docstring")) - continue if prev_test_result.get(item.nodeid) in ("passed", "warnings", "skip_passed"): item.add_marker(pytest.mark.skip(reason="previously passed")) continue diff --git a/docs/sphinx_pyodide/sphinx_pyodide/mdn_xrefs.py b/docs/sphinx_pyodide/sphinx_pyodide/mdn_xrefs.py index 85950d3533f..7b91787dbf3 100644 --- a/docs/sphinx_pyodide/sphinx_pyodide/mdn_xrefs.py +++ b/docs/sphinx_pyodide/sphinx_pyodide/mdn_xrefs.py @@ -19,6 +19,7 @@ "js:class": { "Array": "$global/", "NodeList": "API/", + "XMLHttpRequest": "API/", "HTMLCollection": "API/", "HTMLCanvasElement": "API/", "Generator": "$global/", diff --git a/src/py/_pyodide/_base.py b/src/py/_pyodide/_base.py index 7f9b0ced7d1..7acf433660a 100644 --- a/src/py/_pyodide/_base.py +++ b/src/py/_pyodide/_base.py @@ -215,19 +215,18 @@ class CodeRunner: Examples -------- - >>> from pyodide.code import CodeRunner >>> source = "1 + 1" >>> code_runner = CodeRunner(source) - >>> code_runner.compile() - <_pyodide._base.CodeRunner object at 0x113de58> + >>> code_runner.compile() # doctest: +ELLIPSIS + <_pyodide._base.CodeRunner object at 0x...> >>> code_runner.run() 2 >>> my_globals = {"x": 20} >>> my_locals = {"y": 5} >>> source = "x + y" >>> code_runner = CodeRunner(source) - >>> code_runner.compile() - <_pyodide._base.CodeRunner object at 0x1166bb0> + >>> code_runner.compile() # doctest: +ELLIPSIS + <_pyodide._base.CodeRunner object at 0x...> >>> code_runner.run(globals=my_globals, locals=my_locals) 25 """ @@ -464,7 +463,6 @@ def eval_code( Examples -------- - >>> from pyodide.code import eval_code >>> source = "1 + 1" >>> eval_code(source) 2 @@ -483,9 +481,13 @@ def eval_code( >>> eval_code(source, return_mode="last_expr") >>> eval_code(source, return_mode="none") >>> source = "print(pyodide)" # Pretend this is open('example_of_filename.py', 'r').read() - >>> eval_code(source, filename="example_of_filename.py") # doctest: +SKIP - # Trackback will show where in the file the error happened - # ...File "example_of_filename.py", line 1, in ...NameError: name 'pyodide' is not defined + >>> eval_code(source, filename="example_of_filename.py") + Traceback (most recent call last): + ... + File "example_of_filename.py", line 1, in + print(pyodide) + ^^^^^^^ + NameError: name 'pyodide' is not defined """ return ( CodeRunner( @@ -596,7 +598,6 @@ def find_imports(source: str) -> list[str]: Examples -------- - >>> from pyodide.code import find_imports >>> source = "import numpy as np; import scipy.stats" >>> find_imports(source) ['numpy', 'scipy'] diff --git a/src/py/_pyodide/_core_docs.py b/src/py/_pyodide/_core_docs.py index 8deac1c3848..1ebc21358e9 100644 --- a/src/py/_pyodide/_core_docs.py +++ b/src/py/_pyodide/_core_docs.py @@ -39,7 +39,9 @@ VTco = TypeVar("VTco", covariant=True) # Value type covariant containers. Tcontra = TypeVar("Tcontra", contravariant=True) # Ditto contravariant. -if "IN_PYTEST" not in os.environ: +if "IN_PYTEST" in os.environ: + __name__ = _save_name +else: __name__ = "pyodide.ffi" _js_flags: dict[str, int] = {} @@ -132,7 +134,7 @@ def object_entries(self) -> "JsProxy": Examples -------- - >>> from pyodide.code import run_js + >>> from pyodide.code import run_js # doctest: +RUN_IN_PYODIDE >>> js_obj = run_js("({first: 'aa', second: 22})") >>> entries = js_obj.object_entries() >>> [(key, val) for key, val in entries] @@ -147,10 +149,10 @@ def object_keys(self) -> "JsProxy": Examples -------- - >>> from pyodide.code import run_js - >>> js_obj = run_js("({first: 1, second: 2, third: 3})") # doctest: +SKIP - >>> keys = js_obj.object_keys() # doctest: +SKIP - >>> list(keys) # doctest: +SKIP + >>> from pyodide.code import run_js # doctest: +RUN_IN_PYODIDE + >>> js_obj = run_js("({first: 1, second: 2, third: 3})") + >>> keys = js_obj.object_keys() + >>> list(keys) ['first', 'second', 'third'] """ raise NotImplementedError @@ -161,10 +163,10 @@ def object_values(self) -> "JsProxy": Examples -------- - >>> from pyodide.code import run_js - >>> js_obj = run_js("({first: 1, second: 2, third: 3})") # doctest: +SKIP - >>> values = js_obj.object_values() # doctest: +SKIP - >>> list(values) # doctest: +SKIP + >>> from pyodide.code import run_js # doctest: +RUN_IN_PYODIDE + >>> js_obj = run_js("({first: 1, second: 2, third: 3})") + >>> values = js_obj.object_values() + >>> list(values) [1, 2, 3] """ raise NotImplementedError @@ -191,25 +193,32 @@ def as_object_map(self, *, hereditary: bool = False) -> "JsMutableMap[str, Any]" Examples -------- - .. code-block:: python + >>> from pyodide.code import run_js # doctest: +RUN_IN_PYODIDE + >>> o = run_js("({x : {y: 2}})") + + Normally you have to access the properties of ``o`` as attributes: + + >>> o.x.y + 2 + >>> o["x"] # is not subscriptable + Traceback (most recent call last): + TypeError: 'pyodide.ffi.JsProxy' object is not subscriptable - from pyodide.code import run_js + ``as_object_map`` allows us to access the property with ``getitem``: - o = run_js("({x : {y: 2}})") - # You have to access the properties of o as attributes - assert o.x.y == 2 - with pytest.raises(TypeError): - o["x"] # is not subscriptable + >>> o.as_object_map()["x"].y + 2 - # as_object_map allows us to access the property with getitem - assert o.as_object_map()["x"].y == 2 + The inner object is not subscriptable because ``hereditary`` is ``False``: - with pytest.raises(TypeError): - # The inner object is not subscriptable because hereditary is False. - o.as_object_map()["x"]["y"] + >>> o.as_object_map()["x"]["y"] + Traceback (most recent call last): + TypeError: 'pyodide.ffi.JsProxy' object is not subscriptable - # When hereditary is True, the inner object is also subscriptable - assert o.as_object_map(hereditary=True)["x"]["y"] == 2 + When ``hereditary`` is ``True``, the inner object is also subscriptable: + + >>> o.as_object_map(hereditary=True)["x"]["y"] + 2 """ raise NotImplementedError @@ -413,19 +422,27 @@ def to_file(self, file: IO[bytes] | IO[str], /) -> None: Example ------- - >>> import pytest; pytest.skip() - >>> from js import Uint8Array + >>> from js import Uint8Array # doctest: +RUN_IN_PYODIDE + >>> from pathlib import Path + >>> Path("file.bin").write_text("abc\\x00123ttt") + 10 >>> x = Uint8Array.new(range(10)) >>> with open('file.bin', 'wb') as fh: ... x.to_file(fh) - which is equivalent to, + + This is equivalent to + >>> with open('file.bin', 'wb') as fh: ... data = x.to_bytes() ... fh.write(data) + 10 + but the latter copies the data twice whereas the former only copies the data once. """ + pass + def from_file(self, file: IO[bytes] | IO[str], /) -> None: """Reads from a file into a buffer. @@ -434,20 +451,27 @@ def from_file(self, file: IO[bytes] | IO[str], /) -> None: Example ------- - >>> import pytest; pytest.skip() + >>> None # doctest: +RUN_IN_PYODIDE + >>> from pathlib import Path + >>> Path("file.bin").write_text("abc\\x00123ttt") + 10 >>> from js import Uint8Array >>> # the JsProxy need to be pre-allocated - >>> x = Uint8Array.new(range(10)) + >>> x = Uint8Array.new(10) >>> with open('file.bin', 'rb') as fh: - ... x.read_file(fh) + ... x.from_file(fh) + which is equivalent to + >>> x = Uint8Array.new(range(10)) >>> with open('file.bin', 'rb') as fh: - ... chunk = fh.read(size=x.byteLength) + ... chunk = fh.read(x.byteLength) ... x.assign(chunk) + but the latter copies the data twice whereas the former only copies the data once. """ + pass def _into_file(self, file: IO[bytes] | IO[str], /) -> None: """Will write the entire contents of a buffer into a file using @@ -462,15 +486,21 @@ def _into_file(self, file: IO[bytes] | IO[str], /) -> None: Example ------- - >>> import pytest; pytest.skip() - >>> from js import Uint8Array + >>> from js import Uint8Array # doctest: +RUN_IN_PYODIDE + >>> from pathlib import Path + >>> Path("file.bin").write_text("abc\\x00123ttt") + 10 >>> x = Uint8Array.new(range(10)) >>> with open('file.bin', 'wb') as fh: ... x._into_file(fh) + which is similar to + >>> with open('file.bin', 'wb') as fh: ... data = x.to_bytes() ... fh.write(data) + 10 + but the latter copies the data once whereas the former doesn't copy the data. """ diff --git a/src/py/pyodide/http.py b/src/py/pyodide/http.py index e9c73390b8a..aa9c3102048 100644 --- a/src/py/pyodide/http.py +++ b/src/py/pyodide/http.py @@ -27,8 +27,10 @@ def open_url(url: str) -> StringIO: """Fetches a given URL synchronously. - The download of binary files is not supported. To download binary - files use :func:`pyodide.http.pyfetch` which is asynchronous. + The download of binary files is not supported. To download binary files use + :func:`pyodide.http.pyfetch` which is asynchronous. + + It will not work in Node unless you include an polyfill for :js:class:`XMLHttpRequest`. Parameters ---------- @@ -41,15 +43,14 @@ def open_url(url: str) -> StringIO: Examples -------- - >>> from pyodide.http import open_url - >>> url = "https://cdn.jsdelivr.net/pyodide/v0.23.4/full/repodata.json" + >>> None # doctest: +RUN_IN_PYODIDE + >>> import pytest; pytest.skip("TODO: Figure out how to skip this only in node") + >>> url = "https://cdn.jsdelivr.net/pyodide/v0.24.1/full/pyodide-lock.json" >>> url_contents = open_url(url) - >>> url_contents.read() - { - "info": { - ... # long output truncated - } - } + >>> import json + >>> result = json.load(url_contents) + >>> sorted(list(result["info"].items())) + [('arch', 'wasm32'), ('platform', 'emscripten_3_1_45'), ('python', '3.11.3'), ('version', '0.24.1')] """ req = XMLHttpRequest.new() @@ -289,7 +290,7 @@ async def pyfetch(url: str, **kwargs: Any) -> FetchResponse: Examples -------- - >>> from pyodide.http import pyfetch + >>> import pytest; pytest.skip("Can't use top level await in doctests") >>> res = await pyfetch("https://cdn.jsdelivr.net/pyodide/v0.23.4/full/repodata.json") >>> res.ok True