diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f9b04bb..624aa30 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 17db6db..b8fd240 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1dbe2..3b48197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- New command line arg to `bake`: `--version`. Also switch to relative imports. ([#17]) +- Raise lint dependency versions ([#17]) +- Drop support for python 3.10 and add explicit support for 3.12 ([#17]) + +[#17]: https://github.com/stackabletech/image-tools/pull/17 + + ## 0.0.7 ### Fixed diff --git a/README.md b/README.md index 8d1dcb0..d2ca874 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ pip install git+https://github.com/stackabletech/image-tools.git@main Update the version in: -* `pyproject.toml` +* `src/image_tools/version.py` * `README.md` : version and pip install command. Update the CHANGELOG. diff --git a/pyproject.toml b/pyproject.toml index e99d65c..620b18f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,10 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + [project] name = "image-tools-stackabletech" -version = "0.0.7" +dynamic = ["version"] authors = [{ name = "Razvan Mihai", email = "razvan.mihai@stackable.tech" }] description = "Image tools for the Stackable Data Platform." readme = "README.md" @@ -12,8 +16,8 @@ classifiers = [ ] dependencies = ["Jinja2>=3.1.2", "PyYAML>=6.0"] [project.optional-dependencies] -lint = ['ruff==0.0.275', 'mypy==1.4.0'] -publish = ['twine==5.0.0', 'build==1.2.1'] +lint = ['ruff>=0.4', 'mypy>=1.10'] +publish = ['twine>=5.0', 'build>=1.2'] [project.scripts] bake = "image_tools.bake:main" @@ -29,3 +33,6 @@ line-length = 120 [tool.mypy] ignore_missing_imports = true + +[tool.setuptools.dynamic] +version = { attr = "image_tools.version.__version__" } diff --git a/src/image_tools/args.py b/src/image_tools/args.py index e8199ac..b4b9b38 100644 --- a/src/image_tools/args.py +++ b/src/image_tools/args.py @@ -4,6 +4,9 @@ import importlib.util import sys +from .version import version + + DEFAULT_IMAGE_VERSION_FORMATS = [ re.compile(r"[2-9][0-9]\.[1-9][0-2]?\.\d+(-.+)?"), re.compile(r"0\.0\.0-dev(-.+)?"), @@ -12,30 +15,40 @@ def bake_args() -> Namespace: parser = ArgumentParser( - description="Build and publish product images. Requires docker and buildx (https://github.com/docker/buildx)." + description=f"bake {version()} Build and publish product images. Requires docker and buildx (https://github.com/docker/buildx)." + ) + parser.add_argument("-v", "--version", help="Display version", action="store_true") + + ( + parser.add_argument( + "-c", + "--configuration", + help="Configuration file.", + default="./conf.py", + ), ) - parser.add_argument( - "-c", - "--configuration", - help="Configuration file.", - default="./conf.py", - ), parser.add_argument( "-i", "--image-version", help="Image version", - default='0.0.0-dev', + default="0.0.0-dev", type=check_image_version_format, ) - parser.add_argument("-p", "--product", - help="Product to build images for", action='append') - parser.add_argument("--shard-count", type=positive_int, default=1, - help="Split the build into N shards, which can be built separately. \ - All shards must be built separately, by specifying the --shard-index argument.",) - parser.add_argument("--shard-index", type=positive_int, default=0, - help="Build shard number M out of --shard-count. Shards are zero-indexed.") - parser.add_argument("-u", "--push", help="Push images", - action="store_true") + parser.add_argument("-p", "--product", help="Product to build images for", action="append") + parser.add_argument( + "--shard-count", + type=positive_int, + default=1, + help="Split the build into N shards, which can be built separately. \ + All shards must be built separately, by specifying the --shard-index argument.", + ) + parser.add_argument( + "--shard-index", + type=positive_int, + default=0, + help="Build shard number M out of --shard-count. Shards are zero-indexed.", + ) + parser.add_argument("-u", "--push", help="Push images", action="store_true") parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") parser.add_argument( "-a", @@ -56,16 +69,21 @@ def bake_args() -> Namespace: help="Image registry to publish to. Default: docker.stackable.tech", default="docker.stackable.tech", ) - parser.add_argument( - "--export-tags-file", - help="Write target image tags to a text file. Useful for signing or other follow-up CI steps." - ), + ( + parser.add_argument( + "--export-tags-file", + help="Write target image tags to a text file. Useful for signing or other follow-up CI steps.", + ), + ) result = parser.parse_args() if result.shard_index >= result.shard_count: - raise ValueError("shard index [{}] cannot be greater or equal than shard count [{}]".format( - result.shard_index, result.shard_count)) + raise ValueError( + "shard index [{}] cannot be greater or equal than shard count [{}]".format( + result.shard_index, result.shard_count + ) + ) return result @@ -76,8 +94,7 @@ def positive_int(value) -> int: raise ValueError return ivalue except ValueError: - raise ValueError( - f"Invalid value [{value}]. Must be an integer greater than or equal to zero.") + raise ValueError(f"Invalid value [{value}]. Must be an integer greater than or equal to zero.") def check_image_version_format(image_version) -> str: @@ -113,6 +130,9 @@ def preflight_args() -> Namespace: parser = ArgumentParser( description="Run OpenShift certification checks and submit results to RedHat Partner Connect portal" ) + + parser.add_argument("-v", "--version", help="Display version", action="store_true") + parser.add_argument( "-i", "--image-version", @@ -120,11 +140,8 @@ def preflight_args() -> Namespace: required=True, type=check_image_version_format, ) - parser.add_argument( - "-p", "--product", help="Product to build images for", required=True - ) - parser.add_argument( - "-s", "--submit", help="Submit results", action="store_true") + parser.add_argument("-p", "--product", help="Product to build images for", required=True) + parser.add_argument("-s", "--submit", help="Submit results", action="store_true") parser.add_argument("-d", "--dry", help="Dry run.", action="store_true") parser.add_argument( "-a", @@ -156,12 +173,14 @@ def preflight_args() -> Namespace: help="Name of the preflight program. Default: preflight", default="preflight", ) - parser.add_argument( - "-c", - "--configuration", - help="Configuration file.", - default="./conf.py", - ), + ( + parser.add_argument( + "-c", + "--configuration", + help="Configuration file.", + default="./conf.py", + ), + ) result = parser.parse_args() @@ -175,9 +194,7 @@ def check_architecture_input(architecture: str) -> str: supported_arch = ["linux/amd64", "linux/arm64"] if architecture not in supported_arch: - raise ValueError( - f"Architecture {architecture} not supported. Supported: {supported_arch}" - ) + raise ValueError(f"Architecture {architecture} not supported. Supported: {supported_arch}") return architecture diff --git a/src/image_tools/bake.py b/src/image_tools/bake.py index ad5d932..7affa51 100644 --- a/src/image_tools/bake.py +++ b/src/image_tools/bake.py @@ -6,14 +6,16 @@ python -m image_tools.bake -p opa -i 22.12.0 """ + import sys from typing import List, Dict, Any from argparse import Namespace from subprocess import run import json -from image_tools.lib import Command -from image_tools.args import bake_args, load_configuration +from .lib import Command +from .args import bake_args, load_configuration +from .version import version def build_image_args(version: Dict[str, str], release_version: str): @@ -34,9 +36,7 @@ def build_image_args(version: Dict[str, str], release_version: str): return result -def build_image_tags( - image_name: str, image_version: str, product_version: str -) -> List[str]: +def build_image_tags(image_name: str, image_version: str, product_version: str) -> List[str]: """ Returns a list of --tag command line arguments that are used by the docker build command. @@ -59,11 +59,7 @@ def generate_bakefile(args: Namespace, conf) -> Dict[str, Any]: product_name: str = product["name"] product_targets = {} for version_dict in product.get("versions", []): - product_targets.update( - bakefile_product_version_targets( - args, product_name, version_dict, product_names - ) - ) + product_targets.update(bakefile_product_version_targets(args, product_name, version_dict, product_names)) groups[product_name] = { "targets": list(product_targets.keys()), } @@ -85,10 +81,10 @@ def bakefile_target_name_for_product_version(product_name: str, version: str) -> def bakefile_product_version_targets( - args: Namespace, - product_name: str, - versions: Dict[str, str], - product_names: List[str], + args: Namespace, + product_name: str, + versions: Dict[str, str], + product_names: List[str], ): """ Creates Bakefile targets defining how to build a given product version. @@ -96,8 +92,7 @@ def bakefile_product_version_targets( A product is assumed to depend on another if it defines a `versions` field with the same name as the other product. """ image_name = f"{args.registry}/{args.organization}/{product_name}" - tags = build_image_tags( - image_name, args.image_version, versions["product"]) + tags = build_image_tags(image_name, args.image_version, versions["product"]) build_args = build_image_args(versions, args.image_version) return { @@ -118,15 +113,13 @@ def bakefile_product_version_targets( def targets_for_selector(conf, selected_products: List[str]) -> List[str]: targets = [] - for selected_product in selected_products or (product['name'] for product in conf.products): + for selected_product in selected_products or (product["name"] for product in conf.products): product_name, *versions = selected_product.split("=") - product = next( - (product for product in conf.products if product['name'] == product_name), None) + product = next((product for product in conf.products if product["name"] == product_name), None) if product is None: raise ValueError(f"Requested unknown product [{product_name}]") - for version in versions or (version['product'] for version in product['versions']): - targets.append(bakefile_target_name_for_product_version( - product_name, version)) + for ver in versions or (ver["product"] for ver in product["versions"]): + targets.append(bakefile_target_name_for_product_version(product_name, ver)) return targets @@ -150,7 +143,6 @@ def bake_command(args: Namespace, targets: List[str], bakefile) -> Command: else: target_mode = ["--load"] - return Command( args=[ "docker", @@ -169,12 +161,15 @@ def main() -> int: """Generate a Docker bake file from conf.py and build the given args.product images.""" args = bake_args() + if args.version: + print(version()) + return 0 + conf = load_configuration(args.configuration) bakefile = generate_bakefile(args, conf) - targets = filter_targets_for_shard(targets_for_selector( - conf, args.product), args.shard_count, args.shard_index) + targets = filter_targets_for_shard(targets_for_selector(conf, args.product), args.shard_count, args.shard_index) if not targets: print("No targets match this filter") @@ -188,10 +183,9 @@ def main() -> int: result = run(cmd.args, input=cmd.input, check=True) if args.export_tags_file: - with open(args.export_tags_file, 'w') as tf: + with open(args.export_tags_file, "w") as tf: for t in targets: - tf.writelines( - (f"{t}\n" for t in bakefile["target"][t]["tags"])) + tf.writelines((f"{t}\n" for t in bakefile["target"][t]["tags"])) return result.returncode diff --git a/src/image_tools/preflight.py b/src/image_tools/preflight.py index c0a48e0..7ac0f69 100644 --- a/src/image_tools/preflight.py +++ b/src/image_tools/preflight.py @@ -17,9 +17,9 @@ import sys import logging -from image_tools.args import preflight_args, load_configuration -from image_tools.lib import Command -from image_tools.bake import generate_bakefile +from .args import preflight_args, load_configuration +from .lib import Command +from .bake import generate_bakefile def get_images_for_target(product: str, bakefile: Dict[str, Any]) -> List[str]: @@ -38,12 +38,9 @@ def get_preflight_failures(image_commands: Dict[str, Command]) -> Dict[str, List failures = {} for image, cmd in image_commands.items(): try: - preflight_result = subprocess.run( - cmd.args, input=cmd.input, check=True, stdout=subprocess.PIPE - ) + preflight_result = subprocess.run(cmd.args, input=cmd.input, check=True, stdout=subprocess.PIPE) preflight_json = json.loads(preflight_result.stdout) - failures[image] = preflight_json.get( - "results", {}).get("failed", []) + failures[image] = preflight_json.get("results", {}).get("failed", []) except subprocess.CalledProcessError as error: failures[image] = [error.stderr.decode("utf-8")] except FileNotFoundError: @@ -62,22 +59,24 @@ def preflight_commands(images: List[str], args: Namespace, conf) -> Dict[str, Co cmd_args = [args.executable, "check", "container", img] if args.submit: cmd_args.extend( - ["--loglevel", + [ + "--loglevel", "trace", "--submit", "--pyxis-api-token", args.token, "--certification-project-id", f"ospid-{conf.open_shift_projects[args.product]['id']}", - ] + ] ) if args.architecture: cmd_args.extend( - ["--platform", + [ + "--platform", # this argument value has already been checked against valid values with an expected prefix. # Preflight provides the same "linux/" prefix and so it must be removed here. args.architecture.split("linux/")[1], - ] + ] ) result[img] = Command(args=cmd_args) return result @@ -125,10 +124,7 @@ def main() -> int: if len(img_fails) == 0: logging.info("Image [%s] preflight check successful.", image) else: - logging.error( - "Image [%s] preflight check failures: %s", image, ",".join( - img_fails) - ) + logging.error("Image [%s] preflight check failures: %s", image, ",".join(img_fails)) fail_count = sum(map(len, failures.values())) return fail_count diff --git a/src/image_tools/test_generate_bakefile.py b/src/image_tools/test_generate_bakefile.py new file mode 100644 index 0000000..ab0cb27 --- /dev/null +++ b/src/image_tools/test_generate_bakefile.py @@ -0,0 +1,14 @@ +from image_tools.test import conf +import unittest +import sys + +from image_tools.args import bake_args +from image_tools.bake import generate_bakefile + + +class TestGenerateBakefile(unittest.TestCase): + def test_generate_bakefile(self): + sys.argv = ["test", "-p", "airflow"] + bargs = bake_args() + bakefile = generate_bakefile(bargs, conf) + self.assertIsNotNone(bakefile) diff --git a/src/image_tools/version.py b/src/image_tools/version.py new file mode 100644 index 0000000..d45312c --- /dev/null +++ b/src/image_tools/version.py @@ -0,0 +1,5 @@ +__version__ = "0.0.7" + + +def version() -> str: + return __version__