Compatibility/bundle services (#23)

* Remove dependancies of unzip and juju from host machine

* match url/request/unzip patthrn on each charm*_downloader

* No longer need juju in lxd, determined there was no need for security.priv in lxd, and made 'bundles' a required argument

* handle deprecated situations where a bundle lists the application as services

* extend unit tests to cover both types of bundle app specificiations (application/services)

shrinkwrap.py

@@ -117,12 +117,16 @@ def bundles(self):
 
         return self._cached_bundles
 
+    @staticmethod
+    def apps_or_svcs(_bundle):
+        return _bundle.get("applications") or _bundle.get("services")
+
     @property
     def applications(self):
         return {
-            app_name: self.bundles["bundle.yaml"]["applications"].get(app_name) or app
+            app_name: self.apps_or_svcs(self.bundles["bundle.yaml"]).get(app_name) or app
             for bundle in self.bundles.values()
-            for app_name, app in bundle["applications"].items()
+            for app_name, app in self.apps_or_svcs(bundle).items()
         }
 
     def bundle_download(self):
@@ -488,13 +492,13 @@ def update_app(app_name, app):
     for bundle_name, bundle in charms.bundles.items():
         created_bundle = dict(bundle)
         created_bundle["applications"] = {
-            app_name: update_app(app_name, app) for app_name, app in bundle["applications"].items()
+            app_name: update_app(app_name, app) for app_name, app in BundleDownloader.apps_or_svcs(bundle).items()
         }
 
         with (root / bundle_name).open("w") as fp:
             yaml.safe_dump(created_bundle, fp)
 
-        is_trusted = any(app.get("trust") for app in bundle["applications"].values() if app)
+        is_trusted = any(app.get("trust") for app in BundleDownloader.apps_or_svcs(bundle).values() if app)
         is_overlay = bundle_name != "bundle.yaml"
 
         if is_overlay:

tests/unit/test_bundle.yaml → tests/data/test_bundle.yaml

@@ -5,7 +5,7 @@ machines:
     series: focal
   '1':
     series: focal
-applications:
+{{ apps_or_svcs }}:
   containerd:
     charm: cs:~containers/containerd-160
     resources: {}

tests/unit/conftest.py

@@ -1,8 +1,33 @@
-from tempfile import TemporaryDirectory
+from pathlib import Path
+from types import SimpleNamespace
+
+from jinja2 import FileSystemLoader, Environment
 import pytest
 
+DATA = Path(__file__).parent.parent / "data"
+
+
+@pytest.fixture(params=["applications", "services"])
+def test_bundle(request, tmpdir):
+    templateEnv = Environment(loader=FileSystemLoader(searchpath=DATA))
+    template = templateEnv.get_template("test_bundle.yaml")
+
+    rendered = tmpdir / "test_bundle.yaml"
+    with open(rendered, "w") as f:
+        f.write(template.render({"apps_or_svcs": request.param}))
+    yield SimpleNamespace(file=rendered, apps=request.param)
+
+
+@pytest.fixture
+def test_overlay():
+    yield DATA / "test_overlay.yaml"
+
+
+@pytest.fixture
+def test_container_listing():
+    yield DATA / "test_container_listing.txt"
+
 
-@pytest.fixture()
-def tmp_dir():
-    with TemporaryDirectory() as tp:
-        yield tp
+@pytest.fixture
+def test_charm_config():
+    yield DATA / "test_charm_config.yaml"

tests/unit/test_build_offline_bundle.py

@@ -6,19 +6,18 @@
 import mock
 
 
-def test_build_offline_bundle(tmp_dir):
-    root = Path(tmp_dir)
+def test_build_offline_bundle(tmpdir, test_bundle):
+    root = Path(tmpdir)
     charms = mock.MagicMock(spec_set=BundleDownloader)
     app_name = "etcd"
+    apps = test_bundle.apps
 
-    with (Path(__file__).parent / "test_bundle.yaml").open() as fp:
+    with test_bundle.file.open() as fp:
         whole_bundle = yaml.safe_load(fp)
-        whole_bundle["applications"] = {
-            key: value for key, value in whole_bundle["applications"].items() if key == app_name
-        }
+        whole_bundle[apps] = {key: value for key, value in whole_bundle[apps].items() if key == app_name}
         charms.bundles = {"bundle.yaml": whole_bundle}
 
-    for resource in whole_bundle["applications"][app_name]["resources"]:
+    for resource in whole_bundle[apps][app_name]["resources"]:
         rsc_path = root / "resources" / app_name / resource
         rsc_path.mkdir(parents=True)
         (rsc_path / "any-file-name").touch()

tests/unit/test_bundle_downloader.py

@@ -29,7 +29,7 @@ def mock_cs_downloader():
 @mock.patch("shrinkwrap.requests.get")
 @mock.patch("shrinkwrap.requests.post")
 @mock.patch("shrinkwrap.zipfile.ZipFile")
-def test_charmhub_downloader(mock_zipfile, mock_post, mock_get, tmp_dir):
+def test_charmhub_downloader(mock_zipfile, mock_post, mock_get, tmpdir):
     args = mock.MagicMock()
     args.bundle = "ch:kubernetes-unit-test"
     args.channel = None
@@ -40,7 +40,7 @@ def test_charmhub_downloader(mock_zipfile, mock_post, mock_get, tmp_dir):
     mock_downloaded = mock_zipfile.return_value.extractall.return_value
     mock_get.return_value.content = b"bytes-values"
 
-    downloader = BundleDownloader(tmp_dir, args)
+    downloader = BundleDownloader(tmpdir, args)
     result = downloader.bundle_download()
     assert result is mock_downloaded
     mock_post.assert_called_once_with(
@@ -64,29 +64,29 @@ def test_charmhub_downloader(mock_zipfile, mock_post, mock_get, tmp_dir):
 
 @mock.patch("shrinkwrap.requests.get")
 @mock.patch("shrinkwrap.zipfile.ZipFile")
-def test_charmstore_downloader(mock_zipfile, mock_get, tmp_dir):
+def test_charmstore_downloader(mock_zipfile, mock_get, tmpdir):
     args = mock.MagicMock()
     args.bundle = "cs:kubernetes-unit-test"
     args.overlay = []
 
     mock_downloaded = mock_zipfile.return_value.extractall.return_value
     mock_get.return_value.content = b"bytes-values"
 
-    downloader = BundleDownloader(tmp_dir, args)
+    downloader = BundleDownloader(tmpdir, args)
     result = downloader.bundle_download()
     assert result is mock_downloaded
     mock_get.assert_called_once_with("https://api.jujucharms.com/charmstore/v5/kubernetes-unit-test/archive")
     mock_zipfile.assert_called_once()
     assert isinstance(mock_zipfile.call_args.args[0], BytesIO)
 
 
-def test_bundle_downloader(tmp_dir, mock_ch_downloader, mock_cs_downloader):
+def test_bundle_downloader(tmpdir, mock_ch_downloader, mock_cs_downloader):
     args = mock.MagicMock()
     args.bundle = "cs:kubernetes-unit-test"
     args.overlay = []
-    charms_path = Path(tmp_dir) / "charms"
+    charms_path = Path(tmpdir) / "charms"
 
-    downloader = BundleDownloader(tmp_dir, args)
+    downloader = BundleDownloader(tmpdir, args)
     assert downloader.bundle_path == charms_path / ".bundle"
 
     assert downloader.app_download("etcd", {"charm": "etcd", "channel": "latest/edge"}) == "etcd"
@@ -104,19 +104,19 @@ def test_bundle_downloader(tmp_dir, mock_ch_downloader, mock_cs_downloader):
     mock_cs_downloader.assert_called_once_with("kubernetes-unit-test", downloader.bundle_path)
 
 
-def test_bundle_downloader_properties(tmp_dir, mock_overlay_list):
+def test_bundle_downloader_properties(tmpdir, test_bundle, test_overlay, mock_overlay_list):
     args = mock.MagicMock()
     args.bundle = "cs:kubernetes-unit-test"
     args.overlay = ["test-overlay.yaml"]
-    downloader = BundleDownloader(tmp_dir, args)
+    downloader = BundleDownloader(tmpdir, args)
 
     # mock downloaded already
-    with (Path(__file__).parent / "test_bundle.yaml").open() as fp:
+    with test_bundle.file.open() as fp:
         (downloader.bundle_path / "bundle.yaml").write_text(fp.read())
-    with (Path(__file__).parent / "test_overlay.yaml").open() as fp:
+    with test_overlay.open() as fp:
         (downloader.bundle_path / "test-overlay.yaml").write_text(fp.read())
 
-    assert downloader.bundles["bundle.yaml"]["applications"].keys() == {
+    assert downloader.bundles["bundle.yaml"][test_bundle.apps].keys() == {
         "containerd",
         "easyrsa",
         "etcd",

tests/unit/test_container_downloader.py

@@ -19,9 +19,9 @@ def mock_docker_cmd():
             yield ck
 
 
-def test_container_downloader_no_matches(tmp_dir, mock_requests):
-    downloader = ContainerDownloader(tmp_dir)
-    assert downloader.path == Path(tmp_dir) / "containers"
+def test_container_downloader_no_matches(tmpdir, mock_requests):
+    downloader = ContainerDownloader(tmpdir)
+    assert downloader.path == Path(tmpdir) / "containers"
     mock_requests.return_value.json.return_value = []  # mock github response for empty dir listing
     channel = "latest/stable"
     assert downloader.revisions(channel) == []
@@ -30,9 +30,9 @@ def test_container_downloader_no_matches(tmp_dir, mock_requests):
     assert str(ie.value) == "No revisions matched the channel latest/stable"
 
 
-def test_container_downloader(tmp_dir, mock_requests, mock_docker_cmd):
-    downloader = ContainerDownloader(tmp_dir)
-    assert downloader.path == Path(tmp_dir) / "containers"
+def test_container_downloader(tmpdir, test_container_listing, mock_requests, mock_docker_cmd):
+    downloader = ContainerDownloader(tmpdir)
+    assert downloader.path == Path(tmpdir) / "containers"
     mock_requests.return_value.json.return_value = [
         {
             "name": "README.md",
@@ -62,7 +62,7 @@ def test_container_downloader(tmp_dir, mock_requests, mock_docker_cmd):
             "https://raw.githubusercontent.com/charmed-kubernetes/bundle/master/container-images/v1.18.18.txt",
         ),
     ]
-    with (Path(__file__).parent / "test_container_listing.txt").open() as fp:
+    with test_container_listing.open() as fp:
         mock_requests.return_value.text = fp.read()
     downloader.download(channel)
     mock_docker_cmd.assert_has_calls(

tests/unit/test_download_all.py

@@ -11,24 +11,24 @@
 @mock.patch("shrinkwrap.SnapDownloader.download")
 @mock.patch("shrinkwrap.ResourceDownloader.download")
 @mock.patch("shrinkwrap.ResourceDownloader.list")
-def test_download_method(resource_list, resource_dl, snap_dl, app_dl, tmp_dir):
+def test_download_method(resource_list, resource_dl, snap_dl, app_dl, tmpdir, test_bundle, test_charm_config):
     args = mock.MagicMock()
     args.overlay = []
     args.skip_snaps = False
     args.skip_resources = False
     args.skip_containers = False
-    root = Path(tmp_dir)
+    root = Path(tmpdir)
     app_name = "etcd"
 
-    with (Path(__file__).parent / "test_bundle.yaml").open() as fp:
+    with test_bundle.file.open() as fp:
         (root / "charms" / ".bundle").mkdir(parents=True)
         whole_bundle = yaml.safe_load(fp)
-        whole_bundle["applications"] = {
-            key: value for key, value in whole_bundle["applications"].items() if key == app_name
+        whole_bundle[test_bundle.apps] = {
+            key: value for key, value in whole_bundle[test_bundle.apps].items() if key == app_name
         }
         (root / "charms" / ".bundle" / "bundle.yaml").write_text(yaml.safe_dump(whole_bundle))
 
-    with (Path(__file__).parent / "test_charm_config.yaml").open() as fp:
+    with test_charm_config.open() as fp:
         (root / "charms" / app_name).mkdir(parents=True)
         (root / "charms" / app_name / "config.yaml").write_text(fp.read())
 
@@ -67,4 +67,4 @@ def test_download_method(resource_list, resource_dl, snap_dl, app_dl, tmp_dir):
     resource_list.assert_called_once_with(app_dl.return_value, args.channel)
     snap_dl.assert_called_once()
     resource_dl.assert_called_once()
-    assert (Path(tmp_dir) / "resources" / "etcd" / "etcd" / "etcd.snap").is_symlink()
+    assert (Path(tmpdir) / "resources" / "etcd" / "etcd" / "etcd.snap").is_symlink()

tests/unit/test_downloader.py

@@ -3,9 +3,9 @@
 from shrinkwrap import Downloader
 
 
-def test_downloader(tmp_dir):
-    downloader = Downloader(tmp_dir)
-    assert str(downloader.path) == tmp_dir
+def test_downloader(tmpdir):
+    downloader = Downloader(tmpdir)
+    assert str(downloader.path) == tmpdir
     target = Path("./target")
     assert downloader.to_args(target) == ("", target)
     assert downloader.to_args(target, "testable") == (

tests/unit/test_resources_download.py

@@ -18,8 +18,8 @@ def mock_wget_cmd():
         yield cc
 
 
-def test_resource_downloader(tmp_dir, mock_requests, mock_wget_cmd):
-    downloader = ResourceDownloader(tmp_dir)
+def test_resource_downloader(tmpdir, mock_requests, mock_wget_cmd):
+    downloader = ResourceDownloader(tmpdir)
     assert downloader.path.exists(), "Resource path doesn't exist"
 
     with pytest.raises(NotImplementedError) as ie:
@@ -33,7 +33,7 @@ def test_resource_downloader(tmp_dir, mock_requests, mock_wget_cmd):
     )
 
     target = downloader.mark_download("etcd", "cs:etcd", "snapshot", 0, "snapshot.tar.gz")
-    assert target == Path(tmp_dir) / "resources" / "etcd" / "snapshot" / "snapshot.tar.gz"
+    assert target == Path(tmpdir) / "resources" / "etcd" / "snapshot" / "snapshot.tar.gz"
     assert not target.parent.exists()
 
     downloader.download()

tests/unit/test_snap_downloader.py

@@ -13,8 +13,8 @@ def mock_snap_cmd():
             yield co
 
 
-def test_snap_downloader(tmp_dir, mock_snap_cmd):
-    downloader = SnapDownloader(tmp_dir)
+def test_snap_downloader(tmpdir, mock_snap_cmd):
+    downloader = SnapDownloader(tmpdir)
     assert downloader.empty_snap.exists(), "Empty Snap file doesn't exist"
     mock_snap_cmd.return_value = (
         "Fetching channel map info for jq\n"

tests/unit/test_utils.py

@@ -16,13 +16,13 @@ def test_remove_suffix():
     assert remove_suffix("abc-something", "something") == "abc-"
 
 
-def test_charm_channel(tmp_dir):
-    charm_path = Path(tmp_dir) / "charm" / "etcd"
+def test_charm_channel(tmpdir, test_charm_config, test_bundle):
+    charm_path = Path(tmpdir) / "charm" / "etcd"
     charm_path.mkdir(parents=True)
-    with (Path(__file__).parent / "test_charm_config.yaml").open() as fp:
+    with test_charm_config.open() as fp:
         (charm_path / "config.yaml").write_text(fp.read())
-    with (Path(__file__).parent / "test_bundle.yaml").open() as fp:
+    with test_bundle.file.open() as fp:
         bundle = yaml.safe_load(fp)
-    etcd = bundle["applications"]["etcd"]
+    etcd = bundle[test_bundle.apps]["etcd"]
 
     assert charm_channel(etcd, charm_path) == "3.4/stable"