add totality check

pyproject.toml

@@ -118,10 +118,11 @@ include = [
 
 [tool.pytest.ini_options]
 addopts = [
-    "--pyargs",
-    "jupyterlite_pyodide_kernel",
     "-vv",
     "--ff",
+    # only test as-installed package
+    "--pyargs",
+    "jupyterlite_pyodide_kernel",
     # asyncio
     "--script-launch-mode=subprocess",
     # parallel

src/jupyterlite_pyodide_kernel/addons/pyodide.py

@@ -288,6 +288,9 @@ def check(self, manager: LiteManager):
         for pkg_json in self.get_output_labextension_packages():
             yield from self.check_one_package_json(pkg_json)
 
+        if self.install_on_import and self.pyodide_url:
+            yield from self.check_repodata_totality()
+
     def check_one_config_path(self, config_path: Path):
         """verify the JS and repodata for a single jupyter-lite config"""
         if not config_path.exists():
@@ -342,6 +345,59 @@ def check_one_package_json(self, pkg_json: Path):
             file_dep=[self.package_json_schema, pkg_json],
         )
 
+    def check_repodata_totality(self):
+        """check whether the union of all repodata provides all dependencies."""
+        config_paths = sorted(self.get_output_config_paths())
+        yield dict(
+            name="repo:totality",
+            doc="check if the union of all repodata is complete",
+            file_dep=config_paths,
+            actions=[(self.check_totality, [config_paths])],
+        )
+
+    def check_totality(self, config_paths: List[Path]):
+        local_repo_packages: Dict[str, Any] = {}
+        for config_path in config_paths:
+            plugin_config = self.get_pyodide_settings(config_path)
+            for repo_url in plugin_config.get(REPODATA_URLS, []):
+                url = urllib.parse.urlparse(str(repo_url))
+                if url.scheme:
+                    self.log.warning(
+                        "non-local repodata %s in %s will not be checked",
+                        url,
+                        config_path.relative_to(self.manager.output_dir),
+                    )
+                    continue
+                repo_path = (config_path.parent / url.path).resolve()
+                repo_packages = json.loads(repo_path.read_text(**UTF8))["packages"]
+                local_repo_packages[repo_path.parent] = repo_packages
+
+        resolved = {}
+        for repo, packages in local_repo_packages.items():
+            for package_name, package_info in packages.items():
+                file_name = str(package_info["file_name"])
+                file_url = urllib.parse.urlparse(file_name)
+                if file_url.scheme:
+                    resolved[package_name] = "remote"
+                else:
+                    if (repo / file_url.path).exists():
+                        resolved[package_name] = "local"
+
+        missing_deps = {}
+        for repo, packages in local_repo_packages.items():
+            for package_name, package_info in packages.items():
+                for dep in package_info.get("depends", []):
+                    if dep not in resolved:
+                        missing_deps.setdefault(package_name, []).append(dep)
+
+        if missing_deps:
+            print(json.dumps(missing_deps, **JSON_FMT), flush=True)
+            message = (
+                "Repodata is not self-contained:"
+                f"""Dependencies missing for: {sorted(missing_deps)}"""
+            )
+            raise ValueError(message)
+
     def cache_pyodide(self, path_or_url):
         """copy pyodide to the cache"""
         if re.findall(r"^https?://", path_or_url):

src/jupyterlite_pyodide_kernel/constants.py

@@ -39,6 +39,8 @@
 #: the pyodide index of wheels
 REPODATA_JSON = "repodata.json"
 
+#: extra known dependencies
+REPODATA_EXTRA_DEPENDS = {"ipython": ["sqlite3"]}
 
 #: the observed default environment of pyodide
 PYODIDE_MARKER_ENV = {

src/jupyterlite_pyodide_kernel/tests/test_examples.py

@@ -3,7 +3,12 @@
 import shutil
 import os
 from pathlib import Path
+import json
+import pytest
+
+from jupyterlite_core.constants import UTF8
 
+from jupyterlite_pyodide_kernel.constants import PYPI_WHEELS
 from .conftest import HERE
 
 IN_TREE_EXAMPLES = HERE / "../../../examples"
@@ -15,16 +20,39 @@
     )
 
 
-def test_examples(script_runner, tmp_path):
-    """verity the demo site builds (if it available)"""
+@pytest.fixture
+def an_example_with_tarball(tmp_path, a_pyodide_tarball):
     examples = tmp_path / EXAMPLES.name
     shutil.copytree(EXAMPLES, examples)
+    config_path = examples / "jupyter_lite_config.json"
+    config = json.loads(config_path.read_text(**UTF8))
+    config["PyodideAddon"]["pyodide_url"] = str(a_pyodide_tarball)
+    config_path.write_text(json.dumps(config))
+    return examples
 
-    build = script_runner.run("jupyter", "lite", "build", cwd=str(examples))
-    assert build.success
 
-    build = script_runner.run("jupyter", "lite", "archive", cwd=str(examples))
-    assert build.success
+def test_examples_good(script_runner, an_example_with_tarball):
+    """verify the demo site builds (if it available)"""
+    opts = dict(cwd=str(an_example_with_tarball))
 
-    build = script_runner.run("jupyter", "lite", "check", cwd=str(examples))
+    build = script_runner.run("jupyter", "lite", "build", **opts)
     assert build.success
+
+    archive = script_runner.run("jupyter", "lite", "archive", **opts)
+    assert archive.success
+
+    check = script_runner.run("jupyter", "lite", "check", **opts)
+    assert check.success
+
+
+def test_examples_bad_missing(script_runner, an_example_with_tarball):
+    """verify the demo site check fails for missing deps"""
+    opts = dict(cwd=str(an_example_with_tarball))
+
+    shutil.rmtree(an_example_with_tarball / PYPI_WHEELS)
+
+    check = script_runner.run("jupyter", "lite", "check", **opts)
+    assert not check.success
+    all_out = f"{check.stderr}{check.stdout}"
+    assert "ipython" in all_out, "didn't find the missing dependent"
+    assert "sqlite3" in all_out, "didn't find the missing dependency"

src/jupyterlite_pyodide_kernel/wheel_utils.py

@@ -20,6 +20,7 @@
     ALL_WHL,
     NOARCH_WHL,
     PYODIDE_MARKER_ENV,
+    REPODATA_EXTRA_DEPENDS,
     REPODATA_JSON,
     TOP_LEVEL_TXT,
     WHL_RECORD,
@@ -75,9 +76,11 @@ def get_wheel_repodata(whl_path: Path):
     depnendencies.
     """
     name, version, release = get_wheel_fileinfo(whl_path)
-    depends = get_wheel_depends(whl_path)
-    modules = get_wheel_modules(whl_path)
     normalized_name = get_normalized_name(name)
+    depends = get_wheel_depends(whl_path) + REPODATA_EXTRA_DEPENDS.get(
+        normalized_name, []
+    )
+    modules = get_wheel_modules(whl_path)
     pkg_entry = {
         "name": normalized_name,
         "version": version,