diff --git a/pyproject.toml b/pyproject.toml index 5305e1a7..8a37ee5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,10 +118,11 @@ include = [ [tool.pytest.ini_options] addopts = [ - "--pyargs", - "jupyterlite_pyodide_kernel", "-vv", "--ff", + # only test as-installed package + "--pyargs", + "jupyterlite_pyodide_kernel", # asyncio "--script-launch-mode=subprocess", # parallel diff --git a/src/jupyterlite_pyodide_kernel/addons/pyodide.py b/src/jupyterlite_pyodide_kernel/addons/pyodide.py index 67c20ba8..0f7cf582 100644 --- a/src/jupyterlite_pyodide_kernel/addons/pyodide.py +++ b/src/jupyterlite_pyodide_kernel/addons/pyodide.py @@ -288,6 +288,9 @@ def check(self, manager: LiteManager): for pkg_json in self.get_output_labextension_packages(): yield from self.check_one_package_json(pkg_json) + if self.install_on_import and self.pyodide_url: + yield from self.check_repodata_totality() + def check_one_config_path(self, config_path: Path): """verify the JS and repodata for a single jupyter-lite config""" if not config_path.exists(): @@ -342,6 +345,59 @@ def check_one_package_json(self, pkg_json: Path): file_dep=[self.package_json_schema, pkg_json], ) + def check_repodata_totality(self): + """check whether the union of all repodata provides all dependencies.""" + config_paths = sorted(self.get_output_config_paths()) + yield dict( + name="repo:totality", + doc="check if the union of all repodata is complete", + file_dep=config_paths, + actions=[(self.check_totality, [config_paths])], + ) + + def check_totality(self, config_paths: List[Path]): + local_repo_packages: Dict[str, Any] = {} + for config_path in config_paths: + plugin_config = self.get_pyodide_settings(config_path) + for repo_url in plugin_config.get(REPODATA_URLS, []): + url = urllib.parse.urlparse(str(repo_url)) + if url.scheme: + self.log.warning( + "non-local repodata %s in %s will not be checked", + url, + config_path.relative_to(self.manager.output_dir), + ) + continue + repo_path = (config_path.parent / url.path).resolve() + repo_packages = json.loads(repo_path.read_text(**UTF8))["packages"] + local_repo_packages[repo_path.parent] = repo_packages + + resolved = {} + for repo, packages in local_repo_packages.items(): + for package_name, package_info in packages.items(): + file_name = str(package_info["file_name"]) + file_url = urllib.parse.urlparse(file_name) + if file_url.scheme: + resolved[package_name] = "remote" + else: + if (repo / file_url.path).exists(): + resolved[package_name] = "local" + + missing_deps = {} + for repo, packages in local_repo_packages.items(): + for package_name, package_info in packages.items(): + for dep in package_info.get("depends", []): + if dep not in resolved: + missing_deps.setdefault(package_name, []).append(dep) + + if missing_deps: + print(json.dumps(missing_deps, **JSON_FMT), flush=True) + message = ( + "Repodata is not self-contained:" + f"""Dependencies missing for: {sorted(missing_deps)}""" + ) + raise ValueError(message) + def cache_pyodide(self, path_or_url): """copy pyodide to the cache""" if re.findall(r"^https?://", path_or_url): diff --git a/src/jupyterlite_pyodide_kernel/constants.py b/src/jupyterlite_pyodide_kernel/constants.py index bfaeb5c2..08ce3cad 100644 --- a/src/jupyterlite_pyodide_kernel/constants.py +++ b/src/jupyterlite_pyodide_kernel/constants.py @@ -39,6 +39,8 @@ #: the pyodide index of wheels REPODATA_JSON = "repodata.json" +#: extra known dependencies +REPODATA_EXTRA_DEPENDS = {"ipython": ["sqlite3"]} #: the observed default environment of pyodide PYODIDE_MARKER_ENV = { diff --git a/src/jupyterlite_pyodide_kernel/tests/test_examples.py b/src/jupyterlite_pyodide_kernel/tests/test_examples.py index c18c7c2d..10e4252e 100644 --- a/src/jupyterlite_pyodide_kernel/tests/test_examples.py +++ b/src/jupyterlite_pyodide_kernel/tests/test_examples.py @@ -3,7 +3,12 @@ import shutil import os from pathlib import Path +import json +import pytest + +from jupyterlite_core.constants import UTF8 +from jupyterlite_pyodide_kernel.constants import PYPI_WHEELS from .conftest import HERE IN_TREE_EXAMPLES = HERE / "../../../examples" @@ -15,16 +20,39 @@ ) -def test_examples(script_runner, tmp_path): - """verity the demo site builds (if it available)""" +@pytest.fixture +def an_example_with_tarball(tmp_path, a_pyodide_tarball): examples = tmp_path / EXAMPLES.name shutil.copytree(EXAMPLES, examples) + config_path = examples / "jupyter_lite_config.json" + config = json.loads(config_path.read_text(**UTF8)) + config["PyodideAddon"]["pyodide_url"] = str(a_pyodide_tarball) + config_path.write_text(json.dumps(config)) + return examples - build = script_runner.run("jupyter", "lite", "build", cwd=str(examples)) - assert build.success - build = script_runner.run("jupyter", "lite", "archive", cwd=str(examples)) - assert build.success +def test_examples_good(script_runner, an_example_with_tarball): + """verify the demo site builds (if it available)""" + opts = dict(cwd=str(an_example_with_tarball)) - build = script_runner.run("jupyter", "lite", "check", cwd=str(examples)) + build = script_runner.run("jupyter", "lite", "build", **opts) assert build.success + + archive = script_runner.run("jupyter", "lite", "archive", **opts) + assert archive.success + + check = script_runner.run("jupyter", "lite", "check", **opts) + assert check.success + + +def test_examples_bad_missing(script_runner, an_example_with_tarball): + """verify the demo site check fails for missing deps""" + opts = dict(cwd=str(an_example_with_tarball)) + + shutil.rmtree(an_example_with_tarball / PYPI_WHEELS) + + check = script_runner.run("jupyter", "lite", "check", **opts) + assert not check.success + all_out = f"{check.stderr}{check.stdout}" + assert "ipython" in all_out, "didn't find the missing dependent" + assert "sqlite3" in all_out, "didn't find the missing dependency" diff --git a/src/jupyterlite_pyodide_kernel/wheel_utils.py b/src/jupyterlite_pyodide_kernel/wheel_utils.py index 4d378fc8..620c455f 100644 --- a/src/jupyterlite_pyodide_kernel/wheel_utils.py +++ b/src/jupyterlite_pyodide_kernel/wheel_utils.py @@ -20,6 +20,7 @@ ALL_WHL, NOARCH_WHL, PYODIDE_MARKER_ENV, + REPODATA_EXTRA_DEPENDS, REPODATA_JSON, TOP_LEVEL_TXT, WHL_RECORD, @@ -75,9 +76,11 @@ def get_wheel_repodata(whl_path: Path): depnendencies. """ name, version, release = get_wheel_fileinfo(whl_path) - depends = get_wheel_depends(whl_path) - modules = get_wheel_modules(whl_path) normalized_name = get_normalized_name(name) + depends = get_wheel_depends(whl_path) + REPODATA_EXTRA_DEPENDS.get( + normalized_name, [] + ) + modules = get_wheel_modules(whl_path) pkg_entry = { "name": normalized_name, "version": version,