From 2f1e801f9b5fdcae8a80368f0e2a48a71b6ef711 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Thu, 14 Dec 2023 11:55:30 -0800 Subject: [PATCH] Allow configuring multiple buckets for `process_incoming.py` (#68) The incoming package may now declare which bucket it wants to go in using an identifier, and a mapping from bucket identifiers to actual bucket names can be specified in the new `[common.buckets]` config key. --- Makefile | 4 +++ integration/linux/build/centos-7/Dockerfile | 6 ++++- integration/linux/build/centos-8/Dockerfile | 6 ++++- .../linux/build/debian-bookworm/Dockerfile | 6 ++++- .../linux/build/debian-bullseye/Dockerfile | 6 ++++- .../linux/build/debian-buster/Dockerfile | 6 ++++- integration/linux/build/entrypoint.sh | 4 +++ integration/linux/build/fedora-29/Dockerfile | 6 ++++- .../linux/build/linux-aarch64/Dockerfile | 6 ++++- .../linux/build/linux-x86_64/Dockerfile | 6 ++++- .../linux/build/linuxmusl-aarch64/Dockerfile | 4 +++ .../linux/build/linuxmusl-x86_64/Dockerfile | 4 +++ .../linux/build/rockylinux-9/Dockerfile | 4 +++ .../linux/build/ubuntu-bionic/Dockerfile | 6 ++++- .../linux/build/ubuntu-focal/Dockerfile | 6 ++++- .../linux/build/ubuntu-hirsute/Dockerfile | 6 ++++- .../linux/build/ubuntu-jammy/Dockerfile | 6 ++++- server/process_incoming.py | 26 ++++++++++++++++--- 18 files changed, 102 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 37e71499..cf2e958b 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,10 @@ ifneq ($(PKG_SUBDIST),) EXTRAENV += -e PKG_SUBDIST="$(PKG_SUBDIST)" endif +ifneq ($(PKG_TAGS),) + EXTRAENV += -e PKG_TAGS="$(PKG_TAGS)" +endif + ifneq ($(PKG_INSTALL_REF),) EXTRAENV += -e PKG_INSTALL_REF="$(PKG_INSTALL_REF)" endif diff --git a/integration/linux/build/centos-7/Dockerfile b/integration/linux/build/centos-7/Dockerfile index c3a83c8a..b401f57c 100644 --- a/integration/linux/build/centos-7/Dockerfile +++ b/integration/linux/build/centos-7/Dockerfile @@ -185,7 +185,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/centos-8/Dockerfile b/integration/linux/build/centos-8/Dockerfile index 36ecfd16..778f4a2f 100644 --- a/integration/linux/build/centos-8/Dockerfile +++ b/integration/linux/build/centos-8/Dockerfile @@ -185,7 +185,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-bookworm/Dockerfile b/integration/linux/build/debian-bookworm/Dockerfile index 6f825424..d172925a 100644 --- a/integration/linux/build/debian-bookworm/Dockerfile +++ b/integration/linux/build/debian-bookworm/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-bullseye/Dockerfile b/integration/linux/build/debian-bullseye/Dockerfile index b1d47fb1..e274d4ff 100644 --- a/integration/linux/build/debian-bullseye/Dockerfile +++ b/integration/linux/build/debian-bullseye/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/debian-buster/Dockerfile b/integration/linux/build/debian-buster/Dockerfile index 74c332cc..be9478a6 100644 --- a/integration/linux/build/debian-buster/Dockerfile +++ b/integration/linux/build/debian-buster/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/entrypoint.sh b/integration/linux/build/entrypoint.sh index 59bb1de9..5bde044f 100755 --- a/integration/linux/build/entrypoint.sh +++ b/integration/linux/build/entrypoint.sh @@ -39,6 +39,10 @@ if [ -n "${PKG_SUBDIST}" ]; then extraopts+=" --pkg-subdist=${PKG_SUBDIST}" fi +if [ -n "${PKG_TAGS}" ]; then + extraopts+=" --pkg-tags=${PKG_TAGS}" +fi + if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then extraopts+=" --extra-optimizations" fi diff --git a/integration/linux/build/fedora-29/Dockerfile b/integration/linux/build/fedora-29/Dockerfile index db14ba90..770d4381 100644 --- a/integration/linux/build/fedora-29/Dockerfile +++ b/integration/linux/build/fedora-29/Dockerfile @@ -90,7 +90,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -151,6 +151,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linux-aarch64/Dockerfile b/integration/linux/build/linux-aarch64/Dockerfile index d71f4a68..86c30403 100644 --- a/integration/linux/build/linux-aarch64/Dockerfile +++ b/integration/linux/build/linux-aarch64/Dockerfile @@ -218,7 +218,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linux-x86_64/Dockerfile b/integration/linux/build/linux-x86_64/Dockerfile index b5446825..77fb1c43 100644 --- a/integration/linux/build/linux-x86_64/Dockerfile +++ b/integration/linux/build/linux-x86_64/Dockerfile @@ -218,7 +218,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linuxmusl-aarch64/Dockerfile b/integration/linux/build/linuxmusl-aarch64/Dockerfile index fbc0a63b..8ced1435 100644 --- a/integration/linux/build/linuxmusl-aarch64/Dockerfile +++ b/integration/linux/build/linuxmusl-aarch64/Dockerfile @@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/linuxmusl-x86_64/Dockerfile b/integration/linux/build/linuxmusl-x86_64/Dockerfile index fbc0a63b..8ced1435 100644 --- a/integration/linux/build/linuxmusl-x86_64/Dockerfile +++ b/integration/linux/build/linuxmusl-x86_64/Dockerfile @@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/rockylinux-9/Dockerfile b/integration/linux/build/rockylinux-9/Dockerfile index 299f3ce1..d088f565 100644 --- a/integration/linux/build/rockylinux-9/Dockerfile +++ b/integration/linux/build/rockylinux-9/Dockerfile @@ -148,6 +148,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-bionic/Dockerfile b/integration/linux/build/ubuntu-bionic/Dockerfile index a5561eef..a5a95482 100644 --- a/integration/linux/build/ubuntu-bionic/Dockerfile +++ b/integration/linux/build/ubuntu-bionic/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-focal/Dockerfile b/integration/linux/build/ubuntu-focal/Dockerfile index ef79d5b6..8127a8a1 100644 --- a/integration/linux/build/ubuntu-focal/Dockerfile +++ b/integration/linux/build/ubuntu-focal/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-hirsute/Dockerfile b/integration/linux/build/ubuntu-hirsute/Dockerfile index 56a2a0cc..8272ffcc 100644 --- a/integration/linux/build/ubuntu-hirsute/Dockerfile +++ b/integration/linux/build/ubuntu-hirsute/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/integration/linux/build/ubuntu-jammy/Dockerfile b/integration/linux/build/ubuntu-jammy/Dockerfile index 024d94a6..07c0041b 100644 --- a/integration/linux/build/ubuntu-jammy/Dockerfile +++ b/integration/linux/build/ubuntu-jammy/Dockerfile @@ -139,7 +139,7 @@ RUN cd /usr/local/bin \ && ln -s python3-config python-config # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION 23.2.1 +ENV PYTHON_PIP_VERSION 23.3.1 RUN set -ex; \ \ @@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\ extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\ fi\n\ \n\ +if [ -n "${PKG_TAGS}" ]; then\n\ + extraopts+=" --pkg-tags=${PKG_TAGS}"\n\ +fi\n\ +\n\ if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\ extraopts+=" --extra-optimizations"\n\ fi\n\ diff --git a/server/process_incoming.py b/server/process_incoming.py index 218a5629..ea5533df 100755 --- a/server/process_incoming.py +++ b/server/process_incoming.py @@ -23,6 +23,7 @@ import textwrap import boto3 +import boto3.session import click import filelock import semver @@ -30,6 +31,7 @@ from debian import debian_support +import mypy_boto3_s3 from mypy_boto3_s3 import type_defs as s3types from mypy_boto3_s3 import service_resource as s3 @@ -46,6 +48,7 @@ class CommonConfig(TypedDict): signing_key: str + buckets: dict[str, str] class GenericConfig(TypedDict): @@ -268,7 +271,7 @@ def remove_old( channel: str | None = None, ) -> None: logger.info("remove_old: %s %s %s %s", bucket, prefix, keep, channel) - index: dict[str, dict[str, list[str]]] = {} + index: dict[str, dict[semver.VersionInfo, list[str]]] = {} prefix_str = str(prefix) + "/" for obj in bucket.objects.filter(Prefix=prefix_str): if is_metadata_object(obj.key): @@ -532,19 +535,26 @@ def sync_to_s3( @click.command() @click.option("-c", "--config", default="/etc/genrepo.toml") -@click.option("--bucket", default="edgedb-packages") +@click.option("--bucket") @click.option("--incoming-dir") @click.option("--local-dir") @click.argument("upload_listing") # a single file with a listing of many files def main( config: str, - bucket: str, + bucket: Optional[str], incoming_dir: str, local_dir: str, upload_listing: str, ) -> None: with open(config, "rb") as cf: cfg = cast(Config, tomli.load(cf)) + if "common" not in cfg: + raise ValueError("missing required [common] section in config") + if not cfg["common"].get("buckets"): + cfg["common"]["buckets"] = {} + + if bucket: + cfg["common"]["buckets"]["default"] = bucket os.chdir(incoming_dir) with open(upload_listing) as upload_listing_file: @@ -553,7 +563,7 @@ def main( region = os.environ.get("AWS_REGION", "us-east-2") session = boto3.session.Session(region_name=region) - s3 = session.resource("s3") + s3: mypy_boto3_s3.S3ServiceResource = session.resource("s3") # pyright: ignore for path_str in uploads: path = pathlib.Path(path_str) @@ -578,6 +588,14 @@ def main( metadata = json.loads(metadata_file.read()) repository = metadata.get("repository") + tags = metadata.get("tags") or {} + target_bucket = tags.get("bucket", "default") + bucket = cfg["common"]["buckets"].get(target_bucket) + if bucket is None: + raise RuntimeError( + f"invalid target bucket in metadata: {target_bucket!r}" + f", configure it in genrepo.toml" + ) local_dir_path = pathlib.Path(local_dir) temp_dir_path = pathlib.Path(temp_dir)