hypervisor: add ability to unpack hypervisor specialization package

This allows automatic download and unpacking of a debin package with
the hypervisor specialization without any external tooling

ebcl/tools/hypervisor/config_gen.py

@@ -1,13 +1,18 @@
 import argparse
 import logging
 import os
+
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 import jinja2
 import yaml
 
 from ebcl.common import init_logging, log_exception
+from ebcl.common.config import Config
 from ebcl.common.files import resolve_file
+from ebcl.common.version import VersionDepends
+from ebcl.tools.hypervisor.model_gen import ConfigError
 
 from .schema_loader import BaseModel, FileReadProtocol, Schema, merge_dict
 
@@ -90,6 +95,64 @@ def create_files(self) -> None:
                 outpath.write_text(template.read_text("utf-8"))
 
 
+class SpecializationUnpacker:
+    """
+    Unpacks a debian package with hypervisor specialization
+
+    The property directory returns the path to the files on the local filesystem.
+    They are removed when this object is deleted.
+
+    The path is looked up automatically searching for schema.yaml. It can also be
+    specified in the constructor (path).
+    """
+    _tmp_dir: TemporaryDirectory
+    _config_dir: Path
+
+    def __init__(self, package_name: str, config_file: Path, path: str | None = None) -> None:
+        self._tmp_dir = TemporaryDirectory()
+        self._config_dir = Path(self._tmp_dir.name)
+
+        config = Config(str(config_file), self._tmp_dir.name)
+
+        package = config.proxy.find_package(
+            VersionDepends(package_name, None, None, None, config.arch)
+        )
+        if not package:
+            raise ConfigError(f"Cannot find package {package_name}")
+        package = config.proxy.download_package(config.arch, package)
+        if not package:  # pragma: no cover (should never happen)
+            raise ConfigError(f"Cannot download package {package_name}")
+
+        print(package.local_file)
+        print(package.extract(self._tmp_dir.name, None, use_sudo=False))
+
+        os.system(f"find {self._tmp_dir.name}")
+
+        if path:
+            self._config_dir = self._config_dir / path
+        else:
+            self._config_dir = self.find_schema()
+
+    def find_schema(self) -> Path:
+        """Find a single instance of schema.yaml in the archive"""
+        directories: list[Path] = []
+        for root, _, files in os.walk(self._tmp_dir.name):
+            if "schema.yaml" in files:
+                directories.append(Path(root))
+
+        if len(directories) != 1:
+            raise ConfigError(f"Expected exactly one schema.yaml in specialization package, found {len(directories)}.")
+        return directories[0]
+
+    def __del__(self) -> None:
+        self._tmp_dir.cleanup()
+
+    @property
+    def directory(self) -> Path:
+        """Path to the extension directory on the local filesystem"""
+        return self._config_dir
+
+
 @log_exception(call_exit=True)
 def main() -> None:
     """ Main entrypoint of EBcL hypervisor generator. """
@@ -103,12 +166,23 @@ def main() -> None:
         description='Create the config files for the hypervisor')
     parser.add_argument('-s', '--specialization', type=Path,
                         help='Path to hypervisor specialization directory')
+    parser.add_argument('-p', '--specialization-package', type=str,
+                        help='Name of a debian package that contains the hypervisor specialization')
+    parser.add_argument('--specialization-path', type=str,
+                        help='Path to specialization in package')
+    parser.add_argument('-r', '--repo-config', type=Path,
+                        help='Path to a config file with a repository containing the hypervisor specialization')
     parser.add_argument('config_file', type=Path,
                         help='Path to the YAML configuration file')
     parser.add_argument('output', type=Path,
                         help='Path to the output directory')
     args = parser.parse_args()
 
+    if args.specialization_package:
+        if not args.repo_config or not args.repo_config.exists():
+            parser.error("If a SPECIALIZATION_PACKAGE is specified a REPO_CONFIG must be specified as well")
+        unpacker = SpecializationUnpacker(args.specialization_package, args.repo_config, args.specialization_path)
+        args.specialization = unpacker.directory
     generator = HvFileGenerator(args.config_file, args.output, args.specialization)
     generator.create_files()
 

tests/hypervisor/data/test_packages/.gitignore

@@ -0,0 +1 @@
+tmp

tests/hypervisor/data/test_packages/InRelease

@@ -0,0 +1,8 @@
+Origin: Test Repo
+Label: TestRepo
+Architectures: amd64 arm64
+Description: Test repo
+Date: Wed Dec 11 12:00:00 2024
+SHA256:
+ 3f77a280600be6e2cfb825741278847cc440f156c9012525a3006028b33f4f93 1209 Packages
+ 1b34a7b9d81aa35bb27f6898530988e563c0b064b97b35f069d84742819f8e33 628 Packages.xz

tests/hypervisor/data/test_packages/Makefile

@@ -0,0 +1,65 @@
+names := test-empty test-simple test-multiple test-ebclfsa
+
+files_test-empty := files-empty
+arch_test-empty := arm64
+
+files_test-simple := files-ok
+arch_test-simple := arm64
+
+files_test-multiple := files-ok files-ok2
+arch_test-multiple := arm64
+
+# Reuse files from the example and include too much (expected and config) on purpose!
+files_test-ebclfsa := ../examples/qemu_ebclfsa
+arch_test-ebclfsa := arm64
+
+debs := $(addsuffix .deb,$(names))
+all: $(debs) Packages.xz InRelease
+
+
+define gen_targets
+deb := $(1)
+tmp/$$(deb)/DEBIAN:
+	mkdir -p $$@
+tmp/$$(deb)/DEBIAN/%: %.in | tmp/$$(deb)/DEBIAN
+	sed 's/@@NAME@@/$(1)/; s/@@VERSION@@/$$(VERSION)/; s/@@ARCH@@/$$(ARCH)/;' $$< > $$@
+$$(deb).deb: tmp/$$(deb)/DEBIAN/control
+	$$(foreach f,$(files_$(1)),cp -r $$(f)/* tmp/$$(DEB);)
+	dpkg-deb -b tmp/$$(DEB) $$@
+	rm -rf tmp/$$(DEB)
+$$(deb).deb: VERSION=1.0
+$$(deb).deb: ARCH=$(arch_$(1))
+$$(deb).deb: DEB=$(1)
+endef
+
+#$(foreach name,$(names),$(info $(call gen_targets,$(name))))
+$(foreach name,$(names),$(eval $(call gen_targets,$(name))))
+
+
+
+define IN_RELEASE_HEAD
+Origin: Test Repo
+Label: TestRepo
+Architectures: amd64 arm64
+Description: Test repo
+Date: Wed Dec 11 12:00:00 2024
+SHA256:
+endef
+export IN_RELEASE_HEAD
+
+Packages: $(debs)
+	dpkg-scanpackages -m . /dev/null > Packages
+	sed -i 's~./~~g' Packages
+
+Packages.xz: Packages
+	xz -c $^ > $@
+
+InRelease:
+	echo "$$IN_RELEASE_HEAD" > InRelease
+	echo " $$(sha256sum Packages | cut -d' ' -f1) $$(wc -c Packages)" >> InRelease
+	echo " $$(sha256sum Packages.xz | cut -d' ' -f1) $$(wc -c Packages.xz)" >> InRelease
+
+clean:
+	rm -rf tmp
+	rm -rf *.deb
+	rm -f Packages* InRelease

tests/hypervisor/data/test_packages/Packages

@@ -0,0 +1,48 @@
+Package: test-ebclfsa
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-ebclfsa.deb
+Size: 3404
+MD5sum: f3d0fe80d4f93893cacb89b9ff7f4127
+SHA1: d455184b02b14bcd7d6674cc2b3d00a82c1cc759
+SHA256: 2848affa33fe5a1b56fb59b24da5e1172be10d577b06ff02521e5fa7d97f7685
+Description:
+ Example package
+
+Package: test-empty
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-empty.deb
+Size: 488
+MD5sum: 2bee39e8c94278abbfafd5fe0a7d1653
+SHA1: 6b029df5dea0bbe02dbe5c7aa27155dc9fb66113
+SHA256: f3902ffe162954ab3ae3483efda97fadd6b9f48c9e67b2b77e208008c1eefbaf
+Description:
+ Example package
+
+Package: test-multiple
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-multiple.deb
+Size: 546
+MD5sum: a2a503e734bb180d8e2fe0b7d6a5109a
+SHA1: 65b01ecd9ded1525ac0b0ce9b101f709a8591a2e
+SHA256: 1d6b9a3a0c6f2f6db5b44c7bc7033621a59d70270765e5386cb0a553733b0688
+Description:
+ Example package
+
+Package: test-simple
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-simple.deb
+Size: 518
+MD5sum: 061515d128a6513c2302cd34dfa440a3
+SHA1: a9f1b0360f61994698c04e660fbe150df3c906d6
+SHA256: 3595fc09a69b84d8c4a7dcee5a0a3476bc7aab66178c64e2c3089f2b66ca9fca
+Description:
+ Example package
+

tests/hypervisor/data/test_packages/changelog.in

@@ -0,0 +1,3 @@
+@@NAME@@ (@@VERSION@@) UNRELEASED; urgency=medium
+    nothing
+ -- nobody <nobody@example.com>  Mon, 06 Mar 2017 06:52:08 +0100

tests/hypervisor/data/test_packages/control.in

@@ -0,0 +1,6 @@
+Package: @@NAME@@
+Architecture: @@ARCH@@
+Maintainer: nobody
+Version: @@VERSION@@
+Description:
+ Example package

tests/hypervisor/data/test_packages/files-ok/a/b/schema.yaml

@@ -0,0 +1 @@
+version: 1

tests/hypervisor/data/test_packages/files-ok2/c/d/schema.yaml

@@ -0,0 +1 @@
+version: 1

tests/hypervisor/test_config_gen.py

@@ -3,11 +3,23 @@
 
 from pathlib import Path
 
-from ebcl.tools.hypervisor.config_gen import BaseResolver, main as hv_main
+import yaml
+
+from ebcl.common.types.cpu_arch import CpuArch
+from ebcl.tools.hypervisor.config_gen import BaseResolver, SpecializationUnpacker, main as hv_main
+from ebcl.tools.hypervisor.model_gen import ConfigError
 
 test_data = Path(__file__).parent / "data"
 
 
+def compare_directories(actual: Path, expected: Path) -> None:
+    created_files = sorted(actual.iterdir())
+    expected_files = sorted(expected.iterdir())
+    assert list(map(lambda x: x.name, created_files)) == list(map(lambda x: x.name, expected_files))
+    for act, exp in zip(created_files, expected_files):
+        assert act.read_text() == exp.read_text(), f"{act.name} differs"
+
+
 class TestBaseResolver:
     def test_no_includes(self) -> None:
         assert BaseResolver().load("empty.yaml", test_data) == {}
@@ -29,6 +41,76 @@ def test_missing_include(self) -> None:
             BaseResolver().load("missing_include.yaml", test_data)
 
 
+class TestSpecializationUnpacker:
+    config: Path
+
+    @pytest.fixture(scope="function", autouse=True)
+    def setup(self, tmp_path: Path) -> None:
+        self.config = tmp_path / "config.yaml"
+        with self.config.open("w") as f:
+            yaml.dump(
+                {
+                    'arch': str(CpuArch.ARM64),
+                    'apt_repos': [
+                        {
+                            'apt_repo': "file://" + str(test_data / "test_packages"),
+                            'directory': '.'
+                        }
+                    ]
+                },
+                f
+            )
+
+    def test_missing(self) -> None:
+        with pytest.raises(ConfigError, match="^Cannot find package nonexisting-package$"):
+            SpecializationUnpacker("nonexisting-package", self.config)
+
+    def test_empty(self) -> None:
+        with pytest.raises(ConfigError, match="^Expected exactly one schema.yaml in specialization package, found 0.$"):
+            SpecializationUnpacker("test-empty", self.config)
+
+    def test_simple(self) -> None:
+        unpacker = SpecializationUnpacker("test-simple", self.config)
+        assert unpacker.directory.relative_to(unpacker._tmp_dir.name) == Path("a/b")
+
+        path = Path(unpacker._tmp_dir.name)
+        del unpacker
+        assert path.exists() is False
+
+    def test_multiple(self) -> None:
+        with pytest.raises(ConfigError, match="^Expected exactly one schema.yaml in specialization package, found 2.$"):
+            SpecializationUnpacker("test-multiple", self.config)
+
+    def test_manual(self) -> None:
+        unpacker = SpecializationUnpacker("test-multiple", self.config, "a/b")
+        assert unpacker.directory.relative_to(unpacker._tmp_dir.name) == Path("a/b")
+
+    def test_cli(self, tmp_path: Path, capsys: pytest.CaptureFixture) -> None:
+        sys.argv = [
+            "hypervisor_config",
+            "--specialization-package", "test-ebclfsa",
+            str(test_data / "examples" / "qemu_ebclfsa" / "config.yaml"),
+            str(tmp_path / "out")
+        ]
+        # Missing repo config leads to system exit
+        with pytest.raises(SystemExit):
+            hv_main()
+        assert capsys.readouterr().err.endswith(
+            "If a SPECIALIZATION_PACKAGE is specified a REPO_CONFIG must be specified as well\n"
+        )
+
+        sys.argv = [
+            "hypervisor_config",
+            "--specialization-package", "test-ebclfsa",
+            "--repo-config", str(self.config),
+            str(test_data / "examples" / "qemu_ebclfsa" / "config.yaml"),
+            str(tmp_path / "out")
+        ]
+        hv_main()
+
+        compare_directories(tmp_path / "out", test_data / "examples" / "qemu_ebclfsa" / "expected")
+
+
 real_examples = list((test_data / "examples").iterdir())
 
 
@@ -49,11 +131,7 @@ def test_examples(path: Path, tmp_path: Path) -> None:
         str(tmp_path)
     ]
     if extension_dir.is_dir():
-        sys.argv += ["-s", str(extension_dir)]
+        sys.argv += ["--specialization", str(extension_dir)]
     hv_main()
 
-    created_files = sorted(tmp_path.iterdir())
-    expected_files = sorted(expected_dir.iterdir())
-    assert list(map(lambda x: x.name, created_files)) == list(map(lambda x: x.name, expected_files))
-    for act, exp in zip(created_files, expected_files):
-        assert act.read_text() == exp.read_text(), f"{act.name} differs"
+    compare_directories(tmp_path, expected_dir)