feat: add fetch-libs command (#1665)

charmcraft/application/commands/__init__.py

@@ -30,6 +30,7 @@
 from charmcraft.application.commands.remote import RemoteBuild
 from charmcraft.application.commands.store import (
     # auth
+    FetchLibs,
     LoginCommand,
     LogoutCommand,
     WhoamiCommand,
@@ -102,6 +103,7 @@ def fill_command_groups(app: craft_application.Application) -> None:
             CreateLibCommand,
             PublishLibCommand,
             ListLibCommand,
+            FetchLibs,
             FetchLibCommand,
         ],
     )

charmcraft/application/commands/store.py

@@ -28,9 +28,10 @@
 import zipfile
 from collections.abc import Collection
 from operator import attrgetter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import yaml
+from craft_application import util
 from craft_cli import ArgumentParsingError, emit
 from craft_cli.errors import CraftError
 from craft_parts import Step
@@ -41,7 +42,7 @@
 from tabulate import tabulate
 
 import charmcraft.store.models
-from charmcraft import const, env, parts, utils
+from charmcraft import const, env, errors, parts, utils
 from charmcraft.application.commands.base import CharmcraftCommand
 from charmcraft.models import project
 from charmcraft.store import ImageHandler, LocalDockerdInterface, OCIRegistry, Store
@@ -1510,6 +1511,7 @@ class FetchLibCommand(CharmcraftCommand):
     )
     format_option = True
     always_load_project = True
+    hidden = True
 
     def fill_parser(self, parser):
         """Add own parameters to the general parser."""
@@ -1520,7 +1522,7 @@ def fill_parser(self, parser):
             help="Library to fetch (e.g. charms.mycharm.v2.foo.); optional, default to all",
         )
 
-    def run(self, parsed_args):
+    def run(self, parsed_args: argparse.Namespace) -> None:
         """Run the command."""
         if parsed_args.library:
             local_libs_data = [utils.get_lib_info(full_name=parsed_args.library)]
@@ -1534,10 +1536,9 @@ def run(self, parsed_args):
         to_query = []
         for lib in local_libs_data:
             if lib.lib_id is None:
-                item = {"charm_name": lib.charm_name, "lib_name": lib.lib_name}
+                item = {"charm_name": lib.charm_name, "lib_name": lib.lib_name, "api": lib.api}
             else:
-                item = {"lib_id": lib.lib_id}
-            item["api"] = lib.api
+                item = {"lib_id": lib.lib_id, "api": lib.api}
             to_query.append(item)
         libs_tips = store.get_libraries_tips(to_query)
 
@@ -1617,7 +1618,7 @@ def run(self, parsed_args):
         if parsed_args.format:
             output_data = []
             for lib_data, error_message in full_lib_data:
-                datum = {
+                datum: dict[str, Any] = {
                     "charm_name": lib_data.charm_name,
                     "library_name": lib_data.lib_name,
                     "library_id": lib_data.lib_id,
@@ -1634,6 +1635,94 @@ def run(self, parsed_args):
             emit.message(cli.format_content(output_data, parsed_args.format))
 
 
+class FetchLibs(CharmcraftCommand):
+    """Fetch libraries defined in charmcraft.yaml."""
+
+    name = "fetch-libs"
+    help_msg = "Fetch one or more charm libraries"
+    overview = textwrap.dedent(
+        """
+        Fetch charm libraries defined in charmcraft.yaml.
+
+        For each library in the top-level `charm-libs` key, fetch the latest library
+        version matching those requirements.
+
+        For example:
+        charm-libs:
+          # Fetch lib with API version 0.
+          # If `fetch-libs` is run and a newer minor version is available,
+          # it will be fetched from the store.
+          - lib: postgresql.postgres_client
+            version: "0"
+          # Always fetch precisely version 0.57.
+          - lib: mysql.client
+            version: "0.57"
+        """
+    )
+    format_option = True
+    always_load_project = True
+
+    def run(self, parsed_args: argparse.Namespace) -> None:
+        """Fetch libraries."""
+        store = self._services.store
+        charm_libs = self._services.project.charm_libs
+        if not charm_libs:
+            raise errors.LibraryError(
+                message="No dependent libraries declared in charmcraft.yaml.",
+                resolution="Add a 'charm-libs' section to charmcraft.yaml.",
+                retcode=78,  # EX_CONFIG: configuration error
+            )
+        emit.progress("Getting library metadata from charmhub")
+        libs_metadata = store.get_libraries_metadata_by_name(charm_libs)
+        declared_libs = {lib.lib: lib for lib in charm_libs}
+        missing_store_libs = declared_libs.keys() - libs_metadata.keys()
+        if missing_store_libs:
+            missing_libs_source = [declared_libs[lib].dict() for lib in sorted(missing_store_libs)]
+            libs_yaml = util.dump_yaml(missing_libs_source)
+            raise errors.CraftError(
+                f"Could not find the following libraries on charmhub:\n{libs_yaml}",
+                resolution="Use 'charmcraft list-lib' to check library names and versions.",
+                reportable=False,
+                logpath_report=False,
+            )
+
+        emit.trace(f"Library metadata retrieved: {libs_metadata}")
+        local_libs = {
+            f"{lib.charm_name}.{lib.lib_name}": lib for lib in utils.get_libs_from_tree()
+        }
+        emit.trace(f"Local libraries: {local_libs}")
+
+        downloaded_libs = 0
+        for lib_md in libs_metadata.values():
+            lib_name = f"{lib_md.charm_name}.{lib_md.lib_name}"
+            local_lib = local_libs.get(lib_name)
+            if local_lib and local_lib.content_hash == lib_md.content_hash:
+                emit.debug(
+                    f"Skipping {lib_name} because the same file already exists on "
+                    f"disk (hash: {lib_md.content_hash}). "
+                    "Delete the file and re-run 'charmcraft fetch-libs' to force re-download."
+                )
+                continue
+            lib_name = utils.get_lib_module_name(lib_md.charm_name, lib_md.lib_name, lib_md.api)
+            emit.progress(f"Downloading {lib_name}")
+            lib = store.get_library(
+                charm_name=lib_md.charm_name,
+                library_id=lib_md.lib_id,
+                api=lib_md.api,
+                patch=lib_md.patch,
+            )
+            if lib.content is None:
+                raise errors.CraftError(
+                    f"Store returned no content for '{lib.charm_name}.{lib.lib_name}'"
+                )
+            downloaded_libs += 1
+            lib_path = utils.get_lib_path(lib_md.charm_name, lib_md.lib_name, lib_md.api)
+            lib_path.parent.mkdir(exist_ok=True, parents=True)
+            lib_path.write_text(lib.content)
+
+        emit.message(f"Downloaded {downloaded_libs} charm libraries.")
+
+
 class ListLibCommand(CharmcraftCommand):
     """List all libraries belonging to a charm."""
 

charmcraft/errors.py

@@ -50,7 +50,11 @@ def __init__(
         )
 
 
-class BadLibraryPathError(CraftError):
+class LibraryError(CraftError):
+    """Errors related to charm libraries."""
+
+
+class BadLibraryPathError(LibraryError):
     """Subclass to provide a specific error for a bad library path."""
 
     def __init__(self, path):
@@ -59,7 +63,7 @@ def __init__(self, path):
         )
 
 
-class BadLibraryNameError(CraftError):
+class BadLibraryNameError(LibraryError):
     """Subclass to provide a specific error for a bad library name."""
 
     def __init__(self, name):

charmcraft/models/__init__.py

@@ -24,6 +24,7 @@
 from .project import (
     CharmBuildInfo,
     CharmcraftBuildPlanner,
+    CharmLib,
     CharmcraftProject,
     BasesCharm,
     PlatformCharm,
@@ -48,6 +49,7 @@
     "CharmBuildInfo",
     "CharmcraftBuildPlanner",
     "CharmcraftProject",
+    "CharmLib",
     "BundleMetadata",
     "CharmMetadata",
     "CharmMetadataLegacy",

charmcraft/models/project.py

@@ -17,6 +17,7 @@
 import abc
 import datetime
 import pathlib
+import re
 from collections.abc import Iterable, Iterator
 from typing import (
     Any,
@@ -120,6 +121,76 @@ def _listify_architectures(cls, value: str | list[str]) -> list[str]:
         return value
 
 
+class CharmLib(models.CraftBaseModel):
+    """A Charm library dependency for this charm."""
+
+    lib: str = pydantic.Field(
+        title="Library Path (e.g. my_charm.my_library)",
+        regex=r"[a-z0-9_]+\.[a-z0-9_]+",
+    )
+    version: str = pydantic.Field(
+        title="Version filter for the charm. Either an API version or a specific [api].[patch].",
+        regex=r"[0-9]+(\.[0-9]+)?",
+    )
+
+    @pydantic.validator("lib", pre=True)
+    def _validate_name(cls, value: str) -> str:
+        """Validate the lib field, providing a useful error message on failure."""
+        charm_name, _, lib_name = str(value).partition(".")
+        if not charm_name or not lib_name:
+            raise ValueError(
+                f"Library name invalid. Expected '[charm_name].[lib_name]', got {value!r}"
+            )
+        if not re.fullmatch("[a-z0-9_]+", charm_name):
+            if "-" in charm_name:
+                raise ValueError(
+                    f"Invalid charm name in lib {value!r}. Try replacing hyphens ('-') with underscores ('_')."
+                )
+            raise ValueError(
+                f"Invalid charm name for lib {value!r}. Value {charm_name!r} is invalid."
+            )
+        if not re.fullmatch("[a-z0-9_]+", lib_name):
+            raise ValueError(f"Library name {lib_name!r} is invalid.")
+        return str(value)
+
+    @pydantic.validator("version", pre=True)
+    def _validate_api_version(cls, value: str) -> str:
+        """Validate the API version field, providing a useful error message on failure."""
+        api, *_ = str(value).partition(".")
+        try:
+            int(api)
+        except ValueError:
+            raise ValueError(f"API version not valid. Expected an integer, got {api!r}") from None
+        return str(value)
+
+    @pydantic.validator("version", pre=True)
+    def _validate_patch_version(cls, value: str) -> str:
+        """Validate the optional patch version, providing a useful error message."""
+        api, separator, patch = value.partition(".")
+        if not separator:
+            return value
+        try:
+            int(patch)
+        except ValueError:
+            raise ValueError(
+                f"Patch version not valid. Expected an integer, got {patch!r}"
+            ) from None
+        return value
+
+    @property
+    def api_version(self) -> int:
+        """The API version needed for this library."""
+        return int(self.version.partition(".")[0])
+
+    @property
+    def patch_version(self) -> int | None:
+        """The patch version needed for this library, or None if no patch version is specified."""
+        api, _, patch = self.version.partition(".")
+        if not patch:
+            return None
+        return int(patch)
+
+
 @dataclasses.dataclass
 class CharmBuildInfo(models.BuildInfo):
     """Information about a single build option, with charmcraft-specific info.
@@ -381,6 +452,9 @@ class CharmcraftProject(models.Project, metaclass=abc.ABCMeta):
     contact: None = None
     issues: None = None
     source_code: None = None
+    charm_libs: list[CharmLib] = pydantic.Field(
+        default_factory=list, title="List of libraries to use for this charm"
+    )
 
     # These private attributes are not part of the project model but are attached here
     # because Charmcraft uses this metadata.

charmcraft/services/store.py

@@ -17,14 +17,16 @@
 from __future__ import annotations
 
 import platform
-from collections.abc import Collection, Sequence
+from collections.abc import Collection, Mapping, Sequence
 
 import craft_application
 import craft_store
 from craft_store import models
 
-from charmcraft import const, env, errors
+from charmcraft import const, env, errors, store
+from charmcraft.models import CharmLib
 from charmcraft.store import AUTH_DEFAULT_PERMISSIONS, AUTH_DEFAULT_TTL
+from charmcraft.store.models import Library, LibraryMetadataRequest
 
 
 class BaseStoreService(craft_application.AppService):
@@ -33,6 +35,7 @@ class BaseStoreService(craft_application.AppService):
     This service should be easily adjustable for craft-application.
     """
 
+    ClientClass: type[craft_store.StoreClient] = craft_store.StoreClient
     client: craft_store.StoreClient
     _endpoints: craft_store.endpoints.Endpoints = craft_store.endpoints.CHARMHUB
     _environment_auth: str = const.ALTERNATE_AUTH_ENV_VAR
@@ -73,7 +76,7 @@ def setup(self) -> None:
         """Set up the store service."""
         super().setup()
 
-        self.client = craft_store.StoreClient(
+        self.client = self.ClientClass(
             application_name=self._app.name,
             base_url=self._base_url,
             storage_base_url=self._storage_url,
@@ -159,6 +162,9 @@ def get_credentials(
 class StoreService(BaseStoreService):
     """A Store service specifically for Charmcraft."""
 
+    ClientClass = store.Client
+    client: store.Client  # pyright: ignore[reportIncompatibleVariableOverride]
+
     def set_resource_revisions_architectures(
         self, name: str, resource_name: str, updates: dict[int, list[str]]
     ) -> Collection[models.resource_revision_model.CharmResourceRevision]:
@@ -182,3 +188,42 @@ def set_resource_revisions_architectures(
         )
         new_revisions = self.client.list_resource_revisions(name=name, resource_name=resource_name)
         return [rev for rev in new_revisions if int(rev.revision) in updates]
+
+    def get_libraries_metadata(self, libraries: Sequence[CharmLib]) -> Sequence[Library]:
+        """Get the metadata for one or more charm libraries.
+
+        :param libraries: A sequence of libraries to request.
+        :returns: A sequence of the libraries' metadata in the store.
+        """
+        store_libs = []
+        for lib in libraries:
+            charm_name, _, lib_name = lib.lib.partition(".")
+            store_lib = LibraryMetadataRequest(
+                {
+                    "charm-name": charm_name,
+                    "library-name": lib_name,
+                    "api": lib.api_version,
+                }
+            )
+            if (patch_version := lib.patch_version) is not None:
+                store_lib["patch"] = patch_version
+            store_libs.append(store_lib)
+
+        return self.client.fetch_libraries_metadata(store_libs)
+
+    def get_libraries_metadata_by_name(
+        self, libraries: Sequence[CharmLib]
+    ) -> Mapping[str, Library]:
+        """Get a mapping of [charm_name].[library_name] to the requested libraries."""
+        return {
+            f"{lib.charm_name}.{lib.lib_name}": lib
+            for lib in self.get_libraries_metadata(libraries)
+        }
+
+    def get_library(
+        self, charm_name: str, *, library_id: str, api: int | None = None, patch: int | None = None
+    ) -> Library:
+        """Get a library by charm name and ID from charmhub."""
+        return self.client.get_library(
+            charm_name=charm_name, library_id=library_id, api=api, patch=patch
+        )