chore: merge branch 'main' into work/pydantic-2/merge-main

.github/workflows/tics.yaml

@@ -0,0 +1,71 @@
+name: TICS
+
+on:
+  push:
+    branches:
+    - main
+    # to easy test changes to the workflow
+    - tiobe
+
+jobs:
+  CI:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          echo "::group::apt-get update"
+          sudo apt-get update
+          echo "::endgroup::"
+          echo "::group::apt-get install..."
+          sudo apt-get install -y python3 python3-dev libapt-pkg-dev libyaml-dev
+          echo "::endgroup::"
+          echo "::group::pip install"
+          python -m pip install 'tox<5.0' tox-gh
+          echo "::endgroup::"
+
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          brew install skopeo
+
+      - name: Setup Tox environment
+        run: tox --workdir /tmp/tox run-parallel --parallel auto --parallel-no-spinner --parallel-live --colored yes -e test-py3.10 --notest
+
+      - name: Test with tox
+        run: |
+          echo "::group::skopeo"
+          # From tests.yaml
+          # Ensure the version of skopeo comes from homebrew
+          # This is only necessary until we move to noble.
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          # Allow skopeo to access the contents of /run/containers
+          sudo chmod 777 /run/containers
+          # Add an xdg runtime dir for skopeo to look into for an auth.json file
+          sudo mkdir -p /run/user/$(id -u)
+          sudo chown $USER /run/user/$(id -u)
+          export XDG_RUNTIME_DIR=/run/user/$(id -u)
+          echo "::endgroup::"
+
+          tox --workdir /tmp/tox run --skip-pkg-install --no-list-dependencies --result-json results/tox-py310.json --colored yes -e test-py3.10
+
+      - name: Upload test results
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-ubuntu-22.04
+          path: results/
+
+      - name: Run TICS analysis
+        uses: tiobe/tics-github-action@v3
+        env:
+          PATH: "/tmp/tox/test-py3.10/bin:/snap/bin:/home/runner/.local/bin:/home/runner/.cargo/bin:/bin:/usr/bin:/usr/local/bin:"
+        with:
+          mode: qserver
+          project: charmcraft
+          viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default
+          branchdir: ${{ github.workspace }}
+          ticsAuthToken: ${{ secrets.TICSAUTHTOKEN }}
+          installTics: true

charmcraft/application/main.py

@@ -44,6 +44,15 @@
     summary=GENERAL_SUMMARY,
     ProjectClass=models.CharmcraftProject,
     BuildPlannerClass=models.CharmcraftBuildPlanner,
+    source_ignore_patterns=["*.charm", "charmcraft.yaml"],
+)
+
+PRIME_BEHAVIOUR_CHANGE_MESSAGE = (
+    "IMPORTANT: The behaviour of the 'prime' keyword has changed in Charmcraft 3. This "
+    "keyword will no longer add files that would otherwise be excluded from the "
+    "charm, instead filtering existing files. Additional files may be added using the "
+    "'dump' plugin.\n"
+    "To include extra files, see: https://juju.is/docs/sdk/include-extra-files-in-a-charm"
 )
 
 
@@ -71,23 +80,23 @@ def _project_vars(self, yaml_data: dict[str, Any]) -> dict[str, str]:
 
     def _check_deprecated(self, yaml_data: dict[str, Any]) -> None:
         """Check for deprecated fields in the yaml_data."""
+        # We only need to warn people once.
+        if self.is_managed():
+            return
+        has_primed_part = False
         if "parts" in yaml_data:
-            for k, v in yaml_data["parts"].items():
-                if (k in ("charm", "reactive", "bundle")) or (
-                    v.get("plugin", None) in ("charm", "reactive", "bundle")
-                ):
-                    if "prime" in v:
-                        craft_cli.emit.progress(
-                            "Warning: use of 'prime' in a charm part "
-                            "is deprecated and no longer works, "
-                            "see https://juju.is/docs/sdk/include-extra-files-in-a-charm",
-                            permanent=True,
-                        )
+            prime_changed_extensions = {"charm", "reactive", "bundle"}
+            for name, part in yaml_data["parts"].items():
+                if not {name, part.get("plugin", None)} & prime_changed_extensions:
+                    continue
+                if "prime" in part:
+                    has_primed_part = True
+        if has_primed_part:
+            craft_cli.emit.progress(PRIME_BEHAVIOUR_CHANGE_MESSAGE, permanent=True)
 
     def _extra_yaml_transform(
         self, yaml_data: dict[str, Any], *, build_on: str, build_for: str | None
     ) -> dict[str, Any]:
-        self._check_deprecated(yaml_data)
 
         # Extensions get applied on as close as possible to what the user provided.
         yaml_data = extensions.apply_extensions(self.project_dir, yaml_data.copy())
@@ -99,6 +108,7 @@ def _extra_yaml_transform(
         preprocess.add_actions(self.project_dir, yaml_data)
         preprocess.add_metadata(self.project_dir, yaml_data)
 
+        self._check_deprecated(yaml_data)
         return yaml_data
 
     def _configure_services(self, provider_name: str | None) -> None:

charmcraft/charm_builder.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Canonical Ltd.
+# Copyright 2020-2024 Canonical Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,18 +61,18 @@ def __init__(
         installdir: pathlib.Path,
         entrypoint: pathlib.Path,
         allow_pip_binary: bool = None,
-        binary_python_packages: list[str] = None,
-        python_packages: list[str] = None,
-        requirements: list[pathlib.Path] = None,
+        binary_python_packages: list[str] | None = None,
+        python_packages: list[str] | None = None,
+        requirements: list[pathlib.Path] | None = None,
         strict_dependencies: bool = False,
     ) -> None:
         self.builddir = builddir
         self.installdir = installdir
         self.entrypoint = entrypoint
         self.allow_pip_binary = allow_pip_binary
-        self.binary_python_packages = binary_python_packages
-        self.python_packages = python_packages
-        self.requirement_paths = requirements
+        self.binary_python_packages = binary_python_packages or []
+        self.python_packages = python_packages or []
+        self.requirement_paths = requirements or []
         self.strict_dependencies = strict_dependencies
         self.ignore_rules = self._load_juju_ignore()
         self.ignore_rules.extend_patterns([f"/{const.STAGING_VENV_DIRNAME}"])
@@ -227,7 +227,7 @@ def _calculate_dependencies_hash(self):
         return hashlib.sha1(deps_mashup.encode("utf8")).hexdigest()
 
     @instrum.Timer("Installing dependencies")
-    def _install_dependencies(self, staging_venv_dir):
+    def _install_dependencies(self, staging_venv_dir: pathlib.Path):
         """Install all dependencies in a specific directory."""
         # create virtualenv using the host environment python
         with instrum.Timer("Creating venv"):
@@ -309,6 +309,8 @@ def _install_strict_dependencies(self, pip_cmd: str) -> None:
                 binary_deps=self.binary_python_packages or [],
             )
         )
+        # Validate that the environment is consistent.
+        _process_run([pip_cmd, "check"])
 
     def handle_dependencies(self):
         """Handle from-directory and virtualenv dependencies."""

charmcraft/parts/charm.py

@@ -31,11 +31,6 @@
 from typing_extensions import Self
 
 from charmcraft import charm_builder, env, instrum
-from charmcraft.errors import DependencyError
-from charmcraft.utils import (
-    get_requirements_file_package_names,
-    validate_strict_dependencies,
-)
 
 PACKAGE_NAME_REGEX = re.compile(r"[A-Za-z0-9_.-]+")
 
@@ -93,12 +88,6 @@ def _validate_requirements(self) -> Self:
             )
         project_dirpath = pathlib.Path(self.source)
 
-        # check that all indicated files are present
-        for reqs_filename in self.charm_requirements:
-            reqs_path = project_dirpath / reqs_filename
-            if not reqs_path.is_file():
-                raise ValueError(f"requirements file {str(reqs_path)!r} not found")
-
         # if nothing indicated, and default file is there, use it
         default_reqs_name = "requirements.txt"
         if not self.charm_requirements and (project_dirpath / default_reqs_name).is_file():
@@ -138,18 +127,6 @@ def _validate_strict_dependencies(self) -> Self:
                 f"Invalid package names: {sorted(invalid_binaries)}"
             )
 
-        try:
-            validate_strict_dependencies(
-                get_requirements_file_package_names(
-                    *(pathlib.Path(r) for r in self.charm_requirements)
-                ),
-                self.charm_binary_python_packages,
-            )
-        except DependencyError as e:
-            raise ValueError(
-                "all dependencies must be specified in requirements files for strict dependencies."
-            ) from e
-
         return self
 
 

charmcraft/services/package.py

@@ -17,7 +17,6 @@
 """Service class for packing."""
 from __future__ import annotations
 
-import itertools
 import json
 import os
 import pathlib
@@ -193,7 +192,19 @@ def get_manifest(self, lint_results: Iterable[lint.CheckResult]) -> Manifest:
     def get_manifest_bases(self) -> list[models.Base]:
         """Get the bases used for a charm manifest from the project."""
         if isinstance(self._project, BasesCharm):
-            return list(itertools.chain.from_iterable(base.run_on for base in self._project.bases))
+            run_on_bases = []
+            for project_base in self._project.bases:
+                for build_base in project_base.build_on:
+                    if build_base.name != self._build_plan[0].base.name:
+                        continue
+                    if build_base.channel != self._build_plan[0].base.version:
+                        continue
+                    if self._build_plan[0].build_on not in build_base.architectures:
+                        continue
+                    run_on_bases.extend(project_base.run_on)
+            if not run_on_bases:
+                raise RuntimeError("Could not determine run-on bases.")
+            return run_on_bases
         if isinstance(self._project, PlatformCharm):
             if not self._platform:
                 architectures = [util.get_host_architecture()]

charmcraft/utils/yaml.py

@@ -17,6 +17,7 @@
 
 from typing import Any
 
+import pydantic
 import yaml
 from craft_cli import emit
 
@@ -47,6 +48,9 @@ def _repr_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode:
 def dump_yaml(data: Any) -> str:  # noqa: ANN401: yaml.dump takes anything, so why can't we?
     """Dump a craft model to a YAML string."""
     yaml.add_representer(str, _repr_str, Dumper=yaml.SafeDumper)
+    yaml.add_representer(
+        pydantic.AnyHttpUrl, _repr_str, Dumper=yaml.SafeDumper  # type: ignore[arg-type]
+    )
     yaml.add_representer(
         const.CharmArch,
         yaml.representer.SafeRepresenter.represent_str,

pyproject.toml

@@ -53,6 +53,7 @@ dev = [  # When updating these, also update the dev/lint/types groups in renovat
     "hypothesis",
     "pydocstyle",
     "pyfakefs",
+    "pylint",
     "pytest",
     "pytest-cov",
     "pytest-mock",
@@ -123,7 +124,7 @@ line-length = 99
 
 [tool.codespell]
 ignore-words-list = "buildd,crate,keyserver,comandos,ro,dedent,dedented,tread,socio-economic"
-skip = ".tox,.git,build,.*_cache,__pycache__,*.tar,*.snap,*.png,./node_modules,./docs/_build,.direnv,.venv,venv,.vscode,charmcraft.spec"
+skip = "requirements*.txt,.tox,.git,build,.*_cache,__pycache__,*.tar,*.snap,*.png,./node_modules,./docs/_build,.direnv,.venv,venv,.vscode,charmcraft.spec"
 quiet-level = 3
 check-filenames = true
 

requirements-dev.txt

@@ -1,4 +1,5 @@
 annotated-types==0.7.0
+astroid==3.2.4
 attrs==23.2.0
 certifi==2024.7.4
 cffi==1.16.0
@@ -13,6 +14,8 @@ craft-platforms==0.1.1
 craft-providers==2.0.0
 craft-store==3.0.0
 cryptography==43.0.0
+Deprecated==1.2.14
+dill==0.3.8
 distro==1.9.0
 docker==7.1.0
 flake8==7.1.0
@@ -23,6 +26,7 @@ hypothesis==6.108.5
 idna==3.7
 importlib_metadata==8.2.0
 iniconfig==2.0.0
+isort==5.13.2
 jaraco.classes==3.4.0
 jeepney==0.8.0
 Jinja2==3.1.4
@@ -51,6 +55,7 @@ pydocstyle==6.3.0
 pyfakefs==5.6.0
 pyflakes==3.2.0
 pygit2==1.14.1
+pylint==3.2.6
 pymacaroons==0.13.0
 PyNaCl==1.5.0
 pyparsing==3.1.2
@@ -79,6 +84,9 @@ snap-helpers==0.4.2
 snowballstemmer==2.2.0
 sortedcontainers==2.4.0
 tabulate==0.9.0
+tomlkit==0.13.0
+types-Deprecated==1.2.9.20240311
+types-PyYAML==6.0.12.20240311
 typing_extensions==4.12.2
 urllib3==1.26.19
 wadllib==1.3.6

tests/conftest.py

@@ -45,7 +45,18 @@ def simple_charm():
         name="charmy-mccharmface",
         summary="Charmy!",
         description="Very charming!",
-        bases=[{"name": "ubuntu", "channel": "22.04", "architectures": ["arm64"]}],
+        bases=[
+            {
+                "build-on": [
+                    {
+                        "name": "ubuntu",
+                        "channel": "22.04",
+                        "architectures": [util.get_host_architecture()],
+                    }
+                ],
+                "run-on": [{"name": "ubuntu", "channel": "22.04", "architectures": ["arm64"]}],
+            }
+        ],
     )
 
 
@@ -104,7 +115,7 @@ def default_build_plan():
         models.BuildInfo(
             base=bases.BaseName("ubuntu", "22.04"),
             build_on=arch,
-            build_for=arch,
+            build_for="arm64",
             platform="distro-1-test64",
         )
     ]

tests/integration/services/sample_projects/complex-legacy/prime/manifest.yaml

@@ -13,15 +13,6 @@ bases:
   channel: '22.04'
   architectures:
   - arm64
-- name: ubuntu
-  channel: '20.04'
-  architectures:
-  - amd64
-  - arm64
-  - riscv64
-  - s390x
-  - ppc64el
-  - armhf
 analysis:
   attributes:
   - name: language

tests/integration/services/sample_projects/complex-self-contained/prime/manifest.yaml

@@ -13,15 +13,6 @@ bases:
   channel: '22.04'
   architectures:
   - arm64
-- name: ubuntu
-  channel: '20.04'
-  architectures:
-  - amd64
-  - arm64
-  - riscv64
-  - s390x
-  - ppc64el
-  - armhf
 analysis:
   attributes:
   - name: language