Allow to specify collection_cache in antsibull.cfg

antsibull.cfg

@@ -10,6 +10,9 @@ pypi_url = https://pypi.org/
 thread_max = 15
 max_retries = 10
 file_check_content = 262144
+# Uncomment the following after running `mkdir -p ~/.cache/antsibull/collections`
+# to cache downloaded collection artefacts from Ansible Galaxy in that directory:
+#   collection_cache = ~/.cache/antsibull/collections
 logging_cfg = {
     version = 1.0
     outputs = {

src/antsibull/app_context.py

@@ -68,7 +68,7 @@ def run(args):
 
 #: Field names in the args and config whose value will be added to the app_ctx
 _FIELDS_IN_APP_CTX = frozenset(('ansible_base_url', 'breadcrumbs', 'galaxy_url', 'indexes',
-                                'logging_cfg', 'pypi_url', 'use_html_blobs'))
+                                'logging_cfg', 'pypi_url', 'use_html_blobs', 'collection_cache'))
 
 #: Field names in the args and config whose value will be added to the lib_ctx
 _FIELDS_IN_LIB_CTX = frozenset(

src/antsibull/build_ansible_commands.py

@@ -310,7 +310,7 @@ def rebuild_single_command() -> int:
 
         # Download included collections
         asyncio.run(download_collections(included_versions, app_ctx.galaxy_url,
-                                         download_dir, app_ctx.extra['collection_cache']))
+                                         download_dir, app_ctx.collection_cache))
 
         # Get Ansible changelog, add new release
         ansible_changelog = ChangelogData.ansible(
@@ -321,7 +321,7 @@ def rebuild_single_command() -> int:
             app_ctx.extra['ansible_version'],
             deps_dir=app_ctx.extra['data_dir'],
             deps_data=[dependency_data],
-            collection_cache=app_ctx.extra['collection_cache'],
+            collection_cache=app_ctx.collection_cache,
             ansible_changelog=ansible_changelog)
 
         # Create package and collections directories
@@ -457,7 +457,7 @@ def build_multiple_command() -> int:
         included_versions = asyncio.run(get_collection_versions(deps, app_ctx.galaxy_url))
         asyncio.run(
             download_collections(included_versions, app_ctx.galaxy_url, download_dir,
-                                 app_ctx.extra['collection_cache']))
+                                 app_ctx.collection_cache))
         # TODO: PY3.8:
         # collections_to_install = [p for f in os.listdir(download_dir)
         #                           if os.path.isfile(p := os.path.join(download_dir, f))]

src/antsibull/build_changelog.py

@@ -692,7 +692,7 @@ def build_changelog() -> int:
     ansible_version: PypiVer = app_ctx.extra['ansible_version']
     data_dir: str = app_ctx.extra['data_dir']
     dest_data_dir: str = app_ctx.extra['dest_data_dir']
-    collection_cache: t.Optional[str] = app_ctx.extra['collection_cache']
+    collection_cache: t.Optional[str] = app_ctx.collection_cache
 
     changelog = get_changelog(ansible_version, deps_dir=data_dir, collection_cache=collection_cache)
 

src/antsibull/cli/antsibull_build.py

@@ -198,7 +198,7 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                                          '  Defaults to --data-dir')
 
     cache_parser = argparse.ArgumentParser(add_help=False)
-    cache_parser.add_argument('--collection-cache', default=None,
+    cache_parser.add_argument('--collection-cache', default=argparse.SUPPRESS,
                               help='Directory of cached collection tarballs.  Will be'
                               ' used if a collection tarball to be downloaded exists'
                               ' in here, and will be populated when downloading new'

src/antsibull/cli/antsibull_docs.py

@@ -192,7 +192,7 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                               ' building for. If it is an expanded tarball, the __version__ will'
                               ' be checked to make sure it is compatible with and the same or'
                               ' later version than requested by the deps file.')
-    cache_parser.add_argument('--collection-cache', default=None,
+    cache_parser.add_argument('--collection-cache', default=argparse.SUPPRESS,
                               help='Directory of collection tarballs.  These will be used instead'
                               ' of downloading fresh versions provided that they meet the criteria'
                               ' (Latest version of the collections known to galaxy).')

src/antsibull/cli/doc_commands/devel.py

@@ -105,7 +105,7 @@ def generate_docs() -> int:
             retrieve(collections, tmp_dir,
                      galaxy_server=app_ctx.galaxy_url,
                      ansible_core_source=app_ctx.extra['ansible_core_source'],
-                     collection_cache=app_ctx.extra['collection_cache']))
+                     collection_cache=app_ctx.collection_cache))
         # flog.fields(tarballs=collection_tarballs).debug('Download complete')
         flog.notice('Finished retrieving tarballs')
 

src/antsibull/cli/doc_commands/stable.py

@@ -394,7 +394,7 @@ def generate_docs() -> int:
             retrieve(ansible_core_version, collections, tmp_dir,
                      galaxy_server=app_ctx.galaxy_url,
                      ansible_core_source=app_ctx.extra['ansible_core_source'],
-                     collection_cache=app_ctx.extra['collection_cache']))
+                     collection_cache=app_ctx.collection_cache))
         # flog.fields(tarballs=collection_tarballs).debug('Download complete')
         flog.notice('Finished retrieving tarballs')
 

src/antsibull/schemas/config.py

@@ -11,7 +11,7 @@
 import twiggy.formats
 import twiggy.outputs
 
-from .validators import convert_bool, convert_none
+from .validators import convert_bool, convert_none, convert_path
 
 
 #: Valid choices for a logging level field
@@ -130,7 +130,10 @@ class ConfigModel(BaseModel):
     use_html_blobs: p.StrictBool = False
     thread_max: int = 15
     file_check_content: int = 262144
+    collection_cache: t.Optional[str] = None
 
     _convert_nones = p.validator('process_max', pre=True, allow_reuse=True)(convert_none)
     _convert_bools = p.validator('breadcrumbs', 'indexes', 'use_html_blobs',
                                  pre=True, allow_reuse=True)(convert_bool)
+    _convert_paths = p.validator('collection_cache',
+                                 pre=True, allow_reuse=True)(convert_path)

src/antsibull/schemas/context.py

@@ -8,7 +8,7 @@
 import pydantic as p
 
 from .config import DEFAULT_LOGGING_CONFIG, LoggingModel
-from .validators import convert_bool, convert_none
+from .validators import convert_bool, convert_none, convert_path
 from ..utils.collections import ContextDict
 
 
@@ -54,6 +54,8 @@ class AppContext(BaseModel):
     :ivar indexes: If True, create index pages for all collections and all plugins in a collection.
     :ivar logging_cfg: Configuration of the application logging
     :ivar pypi_url: URL of the pypi server to query for information
+    :ivar collection_cache: If set, must be a path pointing to a directory where collection
+        tarballs are cached so they do not need to be downloaded from Galaxy twice.
     """
 
     extra: ContextDict = ContextDict()
@@ -67,9 +69,12 @@ class AppContext(BaseModel):
     # pyre-ignore[8]: https://github.com/samuelcolvin/pydantic/issues/1684
     pypi_url: p.HttpUrl = 'https://pypi.org/'
     use_html_blobs: p.StrictBool = False
+    collection_cache: t.Optional[str] = None
 
     _convert_bools = p.validator('breadcrumbs', 'indexes', 'use_html_blobs',
                                  pre=True, allow_reuse=True)(convert_bool)
+    _convert_paths = p.validator('collection_cache',
+                                 pre=True, allow_reuse=True)(convert_path)
 
 
 class LibContext(BaseModel):

src/antsibull/schemas/validators.py

@@ -3,6 +3,8 @@
 # License: GPLv3+
 # Copyright: Toshio Kuratomi, 2021
 
+import os.path
+
 
 def convert_none(value):
     """
@@ -48,3 +50,16 @@ def convert_bool(value):
         value = _is_truthy_int(value)
 
     return value
+
+
+def convert_path(value):
+    """
+    Expand `~` and environment variables in strings. Also convert strings like `None` and `Null`
+    to None.
+    """
+    value = convert_none(value)
+
+    if isinstance(value, str):
+        value = os.path.expandvars(os.path.expanduser(value))
+
+    return value