Allow configuring multiple buckets for process_incoming.py (#68)

The incoming package may now declare which bucket it wants to go in using an identifier, and a mapping from bucket identifiers to actual bucket names can be specified in the new `[common.buckets]` config key.
edgedb · Dec 14, 2023 · 2f1e801 · 2f1e801
1 parent 41398f8
commit 2f1e801
Show file tree

Hide file tree

Showing 18 changed files with 102 additions and 16 deletions.
diff --git a/Makefile b/Makefile
@@ -72,6 +72,10 @@ ifneq ($(PKG_SUBDIST),)
  EXTRAENV += -e PKG_SUBDIST="$(PKG_SUBDIST)"
 endif
 
+ifneq ($(PKG_TAGS),)
+ EXTRAENV += -e PKG_TAGS="$(PKG_TAGS)"
+endif
+
 ifneq ($(PKG_INSTALL_REF),)
  EXTRAENV += -e PKG_INSTALL_REF="$(PKG_INSTALL_REF)"
 endif

diff --git a/integration/linux/build/centos-7/Dockerfile b/integration/linux/build/centos-7/Dockerfile
@@ -185,7 +185,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/centos-8/Dockerfile b/integration/linux/build/centos-8/Dockerfile
@@ -185,7 +185,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -246,6 +246,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-bookworm/Dockerfile b/integration/linux/build/debian-bookworm/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-bullseye/Dockerfile b/integration/linux/build/debian-bullseye/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/debian-buster/Dockerfile b/integration/linux/build/debian-buster/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/entrypoint.sh b/integration/linux/build/entrypoint.sh
@@ -39,6 +39,10 @@ if [ -n "${PKG_SUBDIST}" ]; then
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"
 fi
 
+if [ -n "${PKG_TAGS}" ]; then
+ extraopts+=" --pkg-tags=${PKG_TAGS}"
+fi
+
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then
  extraopts+=" --extra-optimizations"
 fi

diff --git a/integration/linux/build/fedora-29/Dockerfile b/integration/linux/build/fedora-29/Dockerfile
@@ -90,7 +90,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -151,6 +151,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linux-aarch64/Dockerfile b/integration/linux/build/linux-aarch64/Dockerfile
@@ -218,7 +218,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linux-x86_64/Dockerfile b/integration/linux/build/linux-x86_64/Dockerfile
@@ -218,7 +218,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -279,6 +279,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linuxmusl-aarch64/Dockerfile b/integration/linux/build/linuxmusl-aarch64/Dockerfile
@@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/linuxmusl-x86_64/Dockerfile b/integration/linux/build/linuxmusl-x86_64/Dockerfile
@@ -82,6 +82,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/rockylinux-9/Dockerfile b/integration/linux/build/rockylinux-9/Dockerfile
@@ -148,6 +148,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-bionic/Dockerfile b/integration/linux/build/ubuntu-bionic/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-focal/Dockerfile b/integration/linux/build/ubuntu-focal/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-hirsute/Dockerfile b/integration/linux/build/ubuntu-hirsute/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/integration/linux/build/ubuntu-jammy/Dockerfile b/integration/linux/build/ubuntu-jammy/Dockerfile
@@ -139,7 +139,7 @@ RUN cd /usr/local/bin \
  && ln -s python3-config python-config
 
 # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value '<VERSION>'"
-ENV PYTHON_PIP_VERSION 23.2.1
+ENV PYTHON_PIP_VERSION 23.3.1
 
 RUN set -ex; \
  \
@@ -200,6 +200,10 @@ if [ -n "${PKG_SUBDIST}" ]; then\n\
  extraopts+=" --pkg-subdist=${PKG_SUBDIST}"\n\
 fi\n\
 \n\
+if [ -n "${PKG_TAGS}" ]; then\n\
+ extraopts+=" --pkg-tags=${PKG_TAGS}"\n\
+fi\n\
+\n\
 if [ -n "${EXTRA_OPTIMIZATIONS}" ]; then\n\
  extraopts+=" --extra-optimizations"\n\
 fi\n\

diff --git a/server/process_incoming.py b/server/process_incoming.py
@@ -23,13 +23,15 @@
 import textwrap
 
 import boto3
+import boto3.session
 import click
 import filelock
 import semver
 import tomli
 
 from debian import debian_support
 
+import mypy_boto3_s3
 from mypy_boto3_s3 import type_defs as s3types
 from mypy_boto3_s3 import service_resource as s3
 
@@ -46,6 +48,7 @@
 
 class CommonConfig(TypedDict):
  signing_key: str
+ buckets: dict[str, str]
 
 
 class GenericConfig(TypedDict):
@@ -268,7 +271,7 @@ def remove_old(
  channel: str | None = None,
 ) -> None:
  logger.info("remove_old: %s %s %s %s", bucket, prefix, keep, channel)
- index: dict[str, dict[str, list[str]]] = {}
+ index: dict[str, dict[semver.VersionInfo, list[str]]] = {}
  prefix_str = str(prefix) + "/"
  for obj in bucket.objects.filter(Prefix=prefix_str):
  if is_metadata_object(obj.key):
@@ -532,19 +535,26 @@ def sync_to_s3(
 
 @click.command()
 @click.option("-c", "--config", default="/etc/genrepo.toml")
-@click.option("--bucket", default="edgedb-packages")
+@click.option("--bucket")
 @click.option("--incoming-dir")
 @click.option("--local-dir")
 @click.argument("upload_listing") # a single file with a listing of many files
 def main(
  config: str,
- bucket: str,
+ bucket: Optional[str],
  incoming_dir: str,
  local_dir: str,
  upload_listing: str,
 ) -> None:
  with open(config, "rb") as cf:
  cfg = cast(Config, tomli.load(cf))
+ if "common" not in cfg:
+ raise ValueError("missing required [common] section in config")
+ if not cfg["common"].get("buckets"):
+ cfg["common"]["buckets"] = {}
+
+ if bucket:
+ cfg["common"]["buckets"]["default"] = bucket
 
  os.chdir(incoming_dir)
  with open(upload_listing) as upload_listing_file:
@@ -553,7 +563,7 @@ def main(
 
  region = os.environ.get("AWS_REGION", "us-east-2")
  session = boto3.session.Session(region_name=region)
- s3 = session.resource("s3")
+ s3: mypy_boto3_s3.S3ServiceResource = session.resource("s3") # pyright: ignore
 
  for path_str in uploads:
  path = pathlib.Path(path_str)
@@ -578,6 +588,14 @@ def main(
 
  metadata = json.loads(metadata_file.read())
  repository = metadata.get("repository")
+ tags = metadata.get("tags") or {}
+ target_bucket = tags.get("bucket", "default")
+ bucket = cfg["common"]["buckets"].get(target_bucket)
+ if bucket is None:
+ raise RuntimeError(
+ f"invalid target bucket in metadata: {target_bucket!r}"
+ f", configure it in genrepo.toml"
+ )
 
  local_dir_path = pathlib.Path(local_dir)
  temp_dir_path = pathlib.Path(temp_dir)