From e11751afa16a8fe2a5d4be4f6e76e6857d547387 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 22 Apr 2024 15:06:57 +0900 Subject: [PATCH 01/63] try snap --- LICENSE | 201 +++++++ pyproject.toml | 16 + snap/snapcraft.yaml | 34 ++ src/github-runner-image-builder/__init__.py | 3 + src/github-runner-image-builder/builder.py | 571 ++++++++++++++++++++ src/github-runner-image-builder/chroot.py | 96 ++++ src/github-runner-image-builder/state.py | 335 ++++++++++++ src/github-runner-image-builder/utils.py | 103 ++++ 8 files changed, 1359 insertions(+) create mode 100644 LICENSE create mode 100644 pyproject.toml create mode 100644 snap/snapcraft.yaml create mode 100644 src/github-runner-image-builder/__init__.py create mode 100644 src/github-runner-image-builder/builder.py create mode 100644 src/github-runner-image-builder/chroot.py create mode 100644 src/github-runner-image-builder/state.py create mode 100644 src/github-runner-image-builder/utils.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b2abfe8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "github-runner-image-builder" +version = "0.0.1" +authors = [{ name = "Yanks Yoon", email = "yangsoo.yoon@canonical.com" }] +description = "A github runner image builder package" +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/canonical/github-runner-image-builder-snap" +Issues = "https://github.com/canonical/github-runner-image-builder-snap/issues" diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 0000000..7cd430c --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,34 @@ +name: github-runner-image-builder +base: core22 # the base snap is the execution environment for this snap +version: "0.1" # just for humans, typically '1.2+git' or '1.3.2' +summary: The snap for building images for github-runners +description: | + Github-runner-image-builder is a snap for building ubuntu images for + github-runner charm (https://github.com/canonical/github-runner-operator/) + used by the github-runner-image-builder charm. It periodically builds + new images according to the configuration. + +license: Apache-2.0 +grade: stable +confinement: strict +architectures: + - build-on: amd64 + build-for: amd64 + - build-on: arm64 + build-for: arm64 + +apps: + github-runner-image-builder: + command: bin/github-runner-image-builder + daemon: simple + +parts: + github-runner-image-builder: + plugin: python + source: . + stage-packages: + - "qemu-utils" + - "libguestfs-tools" + +hooks: + configure: {} diff --git a/src/github-runner-image-builder/__init__.py b/src/github-runner-image-builder/__init__.py new file mode 100644 index 0000000..c5ac4b8 --- /dev/null +++ b/src/github-runner-image-builder/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + diff --git a/src/github-runner-image-builder/builder.py b/src/github-runner-image-builder/builder.py new file mode 100644 index 0000000..697ff60 --- /dev/null +++ b/src/github-runner-image-builder/builder.py @@ -0,0 +1,571 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for interacting with qemu image builder.""" + +import dataclasses +import hashlib +import logging +import os +import shutil + +# Ignore B404:blacklist since all subprocesses are run with predefined executables. +import subprocess # nosec +import urllib.error +import urllib.request +from pathlib import Path +from typing import Literal + +from chroot import ChrootBaseError, ChrootContextManager +from state import Arch, BaseImage +from utils import retry + +logger = logging.getLogger(__name__) + + +class NetworkBlockDeviceError(Exception): + """Represents an error while enabling network block device.""" + + +def _enable_nbd() -> None: + """Enable network block device module to mount and build chrooted image. + + Raises: + NetworkBlockDeviceError: If there was an error enable nbd kernel. + """ + try: + subprocess.run(["/usr/sbin/modprobe", "nbd"], check=True, timeout=10) # nosec: B603 + except subprocess.CalledProcessError as exc: + raise NetworkBlockDeviceError from exc + + +class BuilderSetupError(Exception): + """Represents an error while setting up host machine as builder.""" + + +def setup_builder() -> None: + """Configure the host machine to build images. + + Raises: + BuilderSetupError: If there was an error setting up the host device for building images. + """ + try: + _enable_nbd() + except NetworkBlockDeviceError as exc: + raise BuilderSetupError from exc + + +class UnsupportedArchitectureError(Exception): + """Raised when given machine charm architecture is unsupported. + + Attributes: + arch: The current machine architecture. + """ + + def __init__(self, arch: str) -> None: + """Initialize a new instance of the CharmConfigInvalidError exception. + + Args: + arch: The current machine architecture. + """ + self.arch = arch + + +SupportedCloudImageArch = Literal["amd64", "arm64"] + + +def _get_supported_runner_arch(arch: Arch) -> SupportedCloudImageArch: + """Validate and return supported runner architecture. + + The supported runner architecture takes in arch value from Github supported + architecture and outputs architectures supported by ubuntu cloud images. + See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-\ + self-hosted-runners/about-self-hosted-runners#architectures + and https://cloud-images.ubuntu.com/jammy/current/ + + Args: + arch: The compute architecture to check support for. + + Raises: + UnsupportedArchitectureError: If an unsupported architecture was passed. + + Returns: + The supported architecture. + """ + match arch: + case Arch.X64: + return "amd64" + case Arch.ARM64: + return "arm64" + case _: + raise UnsupportedArchitectureError(arch) + + +IMAGE_MOUNT_DIR = Path("/mnt/ubuntu-image/") +NETWORK_BLOCK_DEVICE_PATH = Path("/dev/nbd0") +NETWORK_BLOCK_DEVICE_PARTITION_PATH = Path("/dev/nbd0p1") + + +def _clean_build_state() -> None: + """Remove any artefacts left by previous build.""" + # The commands will fail if artefacts do not exist and hence there is no need to check the + # output of subprocess runs. + IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False + ) # nosec: B603 + subprocess.run( # nosec: B603 + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False + ) + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], + timeout=30, + check=False, + ) + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + timeout=30, + check=False, + ) + + +CLOUD_IMAGE_URL_TMPL = ( + "https://cloud-images.ubuntu.com/{BASE_IMAGE}/current/" + "{BASE_IMAGE}-server-cloudimg-{BIN_ARCH}.img" +) +CLOUD_IMAGE_FILE_NAME = "{BASE_IMAGE}-server-cloudimg-{BIN_ARCH}.img" + + +class CloudImageDownloadError(Exception): + """Represents an error downloading cloud image.""" + + +def _download_cloud_image(arch: Arch, base_image: BaseImage) -> Path: + """Download the cloud image from cloud-images.ubuntu.com. + + Args: + arch: The cloud image architecture to download. + base_image: The ubuntu base image OS to download. + + Returns: + The downloaded cloud image path. + + Raises: + CloudImageDownloadError: If there was an error downloading the image. + """ + try: + bin_arch = _get_supported_runner_arch(arch) + except UnsupportedArchitectureError as exc: + raise CloudImageDownloadError from exc + + try: + # The ubuntu-cloud-images is a trusted source + image_path, _ = urllib.request.urlretrieve( # nosec: B310 + CLOUD_IMAGE_URL_TMPL.format(BASE_IMAGE=base_image.value, BIN_ARCH=bin_arch), + CLOUD_IMAGE_FILE_NAME.format(BASE_IMAGE=base_image.value, BIN_ARCH=bin_arch), + ) + return Path(image_path) + except urllib.error.ContentTooShortError as exc: + raise CloudImageDownloadError from exc + + +class ImageResizeError(Exception): + """Represents an error while resizing the image.""" + + +def _resize_cloud_img(cloud_image_path: Path) -> None: + """Resize cloud image to allow space for dependency installations. + + Args: + cloud_image_path: The target cloud image file to resize. + + Raises: + ImageResizeError: If there was an error resizing the image. + """ + try: + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-img", "resize", str(cloud_image_path), "+1.5G"], check=True, timeout=60 + ) + except subprocess.CalledProcessError as exc: + raise ImageResizeError from exc + + +class ImageMountError(Exception): + """Represents an error while mounting the image to network block device.""" + + +@retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) +def _mount_nbd_partition() -> None: + """Mount the network block device partition.""" + subprocess.run( # nosec: B603 + [ + "/usr/bin/mount", + "-o", + "rw", + str(NETWORK_BLOCK_DEVICE_PARTITION_PATH), + str(IMAGE_MOUNT_DIR), + ], + check=True, + timeout=60, + ) + + +def _mount_image_to_network_block_device(cloud_image_path: Path) -> None: + """Mount the image to network block device in preparation for chroot. + + Args: + cloud_image_path: The target cloud image file to mount. + + Raises: + ImageMountError: If there was an error mounting the image to network block device. + """ + try: + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", f"--connect={NETWORK_BLOCK_DEVICE_PATH}", str(cloud_image_path)], + check=True, + timeout=60, + ) + _mount_nbd_partition() + except subprocess.CalledProcessError as exc: + raise ImageMountError from exc + + +MOUNTED_RESOLV_CONF_PATH = IMAGE_MOUNT_DIR / "etc/resolv.conf" +HOST_RESOLV_CONF_PATH = Path("/etc/resolv.conf") + + +def _replace_mounted_resolv_conf() -> None: + """Replace resolv.conf to host resolv.conf to allow networking.""" + MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) + shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) + + +class ResizePartitionError(Exception): + """Represents an error while resizing network block device partitions.""" + + +def _resize_mount_partitions() -> None: + """Resize the block partition to fill available space. + + Raises: + ResizePartitionError: If there was an error resizing network block device partitions. + """ + try: + subprocess.run( # nosec: B603 + ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], + check=True, + timeout=60, + ) + subprocess.run( # nosec: B603 + ["/usr/sbin/resize2fs", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + check=True, + timeout=60, + ) + except subprocess.CalledProcessError as exc: + raise ResizePartitionError from exc + + +DEFAULT_PYTHON_PATH = Path("/usr/bin/python3") +SYM_LINK_PYTHON_PATH = Path("/usr/bin/python") + + +def _create_python_symlinks() -> None: + """Create python3 symlinks.""" + os.symlink(DEFAULT_PYTHON_PATH, SYM_LINK_PYTHON_PATH) + + +APT_TIMER = "apt-daily.timer" +APT_SVC = "apt-daily.service" +APT_UPGRADE_TIMER = "apt-daily-upgrade.timer" +APT_UPGRAD_SVC = "apt-daily-upgrade.service" + + +class UnattendedUpgradeDisableError(Exception): + """Represents an error while disabling unattended-upgrade related services.""" + + +def _disable_unattended_upgrades() -> None: + """Disable unatteneded upgrades to prevent apt locks. + + Raises: + UnattendedUpgradeDisableError: If there was an error disabling unattended upgrade related + services. + """ + try: + # use subprocess run rather than operator-libs-linux's systemd library since the library + # does not provide full features like mask. + subprocess.run( + ["/usr/bin/systemctl", "stop", APT_TIMER], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/systemctl", "disable", APT_TIMER], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/systemctl", "mask", APT_SVC], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/systemctl", "stop", APT_UPGRADE_TIMER], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( # nosec: B603 + ["/usr/bin/systemctl", "disable", APT_UPGRADE_TIMER], check=True, timeout=30 + ) + subprocess.run( + ["/usr/bin/systemctl", "mask", APT_UPGRAD_SVC], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/systemctl", "daemon-reload"], check=True, timeout=30 + ) # nosec: B603 + subprocess.run( # nosec: B603 + ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], check=True, timeout=30 + ) + except subprocess.SubprocessError as exc: + raise UnattendedUpgradeDisableError from exc + + +class SystemUserConfigurationError(Exception): + """Represents an error while adding user to chroot env.""" + + +UBUNTU_USER = "ubuntu" +DOCKER_GROUP = "docker" +MICROK8S_GROUP = "microk8s" +UBUNUT_HOME_PATH = Path("/home/ubuntu") + + +def _configure_system_users() -> None: + """Configure system users. + + Raises: + SystemUserConfigurationError: If there was an error configuring ubuntu user. + """ + try: + with (UBUNUT_HOME_PATH / ".profile").open("a") as profile_file: + profile_file.write("PATH=$PATH:/home/ubuntu/.local/bin\n") + subprocess.run( # nosec: B603 + ["/usr/sbin/useradd", "-m", UBUNTU_USER], check=True, timeout=30 + ) + subprocess.run( # nosec: B603 + ["/usr/sbin/groupadd", MICROK8S_GROUP], check=True, timeout=30 + ) + subprocess.run( # nosec: B603 + ["/usr/sbin/usermod", "-aG", DOCKER_GROUP, UBUNTU_USER], check=True, timeout=30 + ) + subprocess.run( # nosec: B603 + ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], check=True, timeout=30 + ) + subprocess.run( # nosec: B603 + ["/usr/bin/chmod", "777", "/usr/local/bin"], check=True, timeout=30 + ) + except subprocess.SubprocessError as exc: + raise SystemUserConfigurationError from exc + + +YQ_DOWNLOAD_URL_TMPL = ( + "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_{BIN_ARCH}" +) +YQ_BINARY_CHECKSUM_URL = "https://github.com/mikefarah/yq/releases/latest/download/checksums" +YQ_CHECKSUM_HASHES_ORDER_URL = ( + "https://github.com/mikefarah/yq/releases/latest/download/checksums_hashes_order" +) +YQ_EXTRACT_CHECKSUM_SCRIPT_URL = ( + "https://github.com/mikefarah/yq/releases/latest/download/extract-checksum.sh" +) + + +class ExternalPackageInstallError(Exception): + """Represents an error installilng external packages.""" + + +def _validate_checksum(file: Path, expected_checksum: str) -> bool: + """Validate the checksum of a given file. + + Args: + file: The file to calculate checksum for. + expected_checksum: The expected file checksum. + + Returns: + True if the checksums match. False otherwise. + """ + sha256 = hashlib.sha256() + sha256.update(file.read_bytes()) + return sha256.hexdigest() == expected_checksum + + +BIN_ARCH_MAP: dict[Arch, str] = {Arch.ARM64: "arm64", Arch.X64: "amd64"} + + +def _install_external_packages(arch: Arch) -> None: + """Install packages outside of apt. + + Installs yarn, yq. + + Args: + arch: The architecture to download binaries for. #TODO check bin arch + + Raises: + ExternalPackageInstallError: If there was an error installing external package. + """ + try: + subprocess.run( + ["/usr/bin/npm", "install", "--global", "yarn"], check=True, timeout=60 * 5 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/npm", "cache", "clean", "--force"], check=True, timeout=60 + ) # nosec: B603 + bin_arch = BIN_ARCH_MAP[arch] + yq_path_str = f"yq_linux_{bin_arch}" + # The URLs are trusted + urllib.request.urlretrieve( + YQ_DOWNLOAD_URL_TMPL.format(BIN_ARCH=bin_arch), yq_path_str + ) # nosec: B310 + urllib.request.urlretrieve(YQ_BINARY_CHECKSUM_URL, "checksums") # nosec: B310 + urllib.request.urlretrieve( + YQ_CHECKSUM_HASHES_ORDER_URL, "checksums_hashes_order" + ) # nosec: B310 + urllib.request.urlretrieve( + YQ_EXTRACT_CHECKSUM_SCRIPT_URL, "extract-checksum.sh" + ) # nosec: B310 + # The output is + checksum = subprocess.check_output( # nosec: B603 + ["/usr/bin/bash", "extract-checksum.sh", "SHA-256", yq_path_str], + encoding="utf-8", + timeout=60, + ).split()[1] + yq_path = Path(yq_path_str) + if not _validate_checksum(yq_path, checksum): + raise ExternalPackageInstallError("Invalid checksum") + yq_path.chmod(755) + yq_path.rename("/usr/bin/yq") + except (subprocess.SubprocessError, urllib.error.ContentTooShortError) as exc: + raise ExternalPackageInstallError from exc + + +class ImageCompressError(Exception): + """Represents an error while compressing cloud-img.""" + + +@retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) +def _compress_image(image: Path) -> Path: + """Compress the cloud image. + + Args: + image: The image to compress. + + Raises: + ImageCompressError: If there was something wrong compressing the image. + + Returns: + The compressed image path. + """ + try: + subprocess.run( # nosec: B603 + ["/usr/bin/virt-sparsify", "--compress", str(image), "compressed.img"], + check=True, + timeout=60 * 10, + ) + return Path("compressed.img") + except subprocess.CalledProcessError as exc: + raise ImageCompressError from exc + + +IMAGE_DEFAULT_APT_PACKAGES = [ + "docker.io", + "npm", + "python3-pip", + "shellcheck", + "jq", + "wget", + "unzip", + "gh", +] + + +@dataclasses.dataclass +class BuildImageConfig: + """Configuration for building the image. + + Attributes: + arch: The CPU architecture to build the image for. + base_image: The ubuntu image to use as build base. + """ + + arch: Arch + base_image: BaseImage + + +class BuildImageError(Exception): + """Represents an error while building the image.""" + + +def build_image(config: BuildImageConfig) -> Path: + """Build and save the image locally. + + Args: + config: The configuration values to build the image with. + + Raises: + BuildImageError: If there was an error building the image. + + Returns: + The saved image path. + """ + logger.info("Clean build state.") + _clean_build_state() + try: + logger.info("Downloading cloud image.") + cloud_image_path = _download_cloud_image(arch=config.arch, base_image=config.base_image) + logger.info("Resizing cloud image.") + _resize_cloud_img(cloud_image_path=cloud_image_path) + logger.info("Mounting network block device.") + _mount_image_to_network_block_device(cloud_image_path=cloud_image_path) + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() + logger.info("Resizing partitions.") + _resize_mount_partitions() + except (CloudImageDownloadError, ImageMountError, ResizePartitionError) as exc: + raise BuildImageError from exc + + try: + logger.info("Setting up chroot environment.") + with ChrootContextManager(IMAGE_MOUNT_DIR): + # operator_libs_linux apt package uses dpkg -l and that does not work well with chroot + # env, hence use subprocess run. + subprocess.run( + ["/usr/bin/apt-get", "update", "-y"], check=True, timeout=60 * 5 + ) # nosec: B603 + subprocess.run( # nosec: B603 + ["/usr/bin/apt-get", "install", "-y", *IMAGE_DEFAULT_APT_PACKAGES], + check=True, + timeout=60 * 10, + ) + _create_python_symlinks() + _disable_unattended_upgrades() + _configure_system_users() + _install_external_packages(arch=config.arch) + except ( + ChrootBaseError, + subprocess.CalledProcessError, + UnattendedUpgradeDisableError, + SystemUserConfigurationError, + ExternalPackageInstallError, + ) as exc: + raise BuildImageError from exc + + try: + _clean_build_state() + logger.info("Compressing image") + return _compress_image(cloud_image_path) + except ImageCompressError as exc: + raise BuildImageError from exc diff --git a/src/github-runner-image-builder/chroot.py b/src/github-runner-image-builder/chroot.py new file mode 100644 index 0000000..7a6beee --- /dev/null +++ b/src/github-runner-image-builder/chroot.py @@ -0,0 +1,96 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Context manager for chrooting.""" + +import os + +# The subprocess calls are from predefined inputs. +import subprocess # nosec: B404 +from pathlib import Path +from typing import Any, cast + +CHROOT_DEVICE_DIR = "dev" +CHROOT_SHARED_DIRS = ["proc", "sys"] +CHROOT_EXTENDED_SHARED_DIRS = [*CHROOT_SHARED_DIRS, "dev"] + + +class ChrootBaseError(Exception): + """Represents the errors with chroot.""" + + +class MountError(ChrootBaseError): + """Represents an error while (un)mounting shared dirs.""" + + +class SyncError(ChrootBaseError): + """Represents an error while syncing chroot dir.""" + + +class ChrootContextManager: + """A helper class for managing chroot environments.""" + + def __init__(self, chroot_path: Path): + """Initialize the chroot context manager. + + Args: + chroot_path: The path to set as new root. + """ + self.chroot_path = chroot_path + self.root: None | int = None + self.cwd: str = "" + + def __enter__(self) -> None: + """Context enter for chroot. + + Raises: + MountError: If there was an error mounting required shared directories. + """ + self.root = os.open("/", os.O_PATH) + self.cwd = os.getcwd() + + for shared_dir in CHROOT_EXTENDED_SHARED_DIRS: + chroot_shared_dir = self.chroot_path / shared_dir + try: + subprocess.run( # nosec: B603 + ["/usr/bin/mount", "--bind", f"/{shared_dir}", str(chroot_shared_dir)], + check=True, + timeout=30, + ) + except subprocess.CalledProcessError as exc: + raise MountError from exc + os.chroot(self.chroot_path) + os.chdir("/") + + def __exit__(self, *_args: Any, **_kwargs: Any) -> None: + """Exit and unmount system dirs. + + Raises: + MountError: if there was an error unmounting shared directories. + SyncError: if there was an error syncing data. + """ + os.chdir(cast(int, self.root)) + os.chroot(".") + os.chdir(self.cwd) + os.close(cast(int, self.root)) + + try: + subprocess.run(["/usr/bin/sync"], check=True) # nosec: B603 + except subprocess.CalledProcessError as exc: + raise SyncError from exc + + for shared_dir in CHROOT_SHARED_DIRS: + chroot_shared_dir = self.chroot_path / shared_dir + try: + subprocess.run( + ["/usr/bin/umount", str(chroot_shared_dir)], check=True + ) # nosec: B603 + except subprocess.CalledProcessError as exc: + raise MountError from exc + + try: + subprocess.run( # nosec: B603 + ["/usr/bin/umount", "-l", str(self.chroot_path / CHROOT_DEVICE_DIR)], check=True + ) + except subprocess.CalledProcessError as exc: + raise MountError from exc diff --git a/src/github-runner-image-builder/state.py b/src/github-runner-image-builder/state.py new file mode 100644 index 0000000..0397163 --- /dev/null +++ b/src/github-runner-image-builder/state.py @@ -0,0 +1,335 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for interacting with charm state and configurations.""" + +import dataclasses +import logging +import os +import platform +from enum import Enum +from typing import Any, Optional + +import yaml +from ops import CharmBase + +logger = logging.getLogger(__name__) + +BASE_IMAGE_CONFIG_NAME = "base-image" +BUILD_INTERVAL_CONFIG_NAME = "build-interval" +OPENSTACK_CLOUDS_YAML_CONFIG_NAME = "experimental-openstack-clouds-yaml" +REVISION_HISTORY_LIMIT_CONFIG_NAME = "revision-history-limit" + + +class Arch(str, Enum): + """Supported system architectures. + + Attributes: + ARM64: Represents an ARM64 system architecture. + X64: Represents an X64/AMD64 system architecture. + """ + + def __str__(self) -> str: + """Interpolate to string value. + + Returns: + The enum string value. + """ + return self.value + + ARM64 = "arm64" + X64 = "x64" + + +class UnsupportedArchitectureError(Exception): + """Raised when given machine charm architecture is unsupported. + + Attributes: + arch: The current machine architecture. + """ + + def __str__(self) -> str: + """Represent the error in string format. + + Returns: + The error in string format. + """ + return f"UnsupportedArchitectureError: {self.arch}" + + def __init__(self, arch: str) -> None: + """Initialize a new instance of the CharmConfigInvalidError exception. + + Args: + arch: The current machine architecture. + """ + self.arch = arch + + +ARCHITECTURES_ARM64 = {"aarch64", "arm64"} +ARCHITECTURES_X86 = {"x86_64"} + + +def _get_supported_arch() -> Arch: + """Get current machine architecture. + + Raises: + UnsupportedArchitectureError: if the current architecture is unsupported. + + Returns: + Arch: Current machine architecture. + """ + arch = platform.machine() + match arch: + case arch if arch in ARCHITECTURES_ARM64: + return Arch.ARM64 + case arch if arch in ARCHITECTURES_X86: + return Arch.X64 + case _: + raise UnsupportedArchitectureError(arch=arch) + + +LTS_IMAGE_VERSION_TAG_MAP = {"22.04": "jammy", "24.04": "noble"} + + +class BaseImage(str, Enum): + """The ubuntu OS base image to build and deploy runners on. + + Attributes: + JAMMY: The jammy ubuntu LTS image. + NOBLE: The noble ubuntu LTS image. + """ + + JAMMY = "jammy" + NOBLE = "noble" + + def __str__(self) -> str: + """Interpolate to string value. + + Returns: + The enum string value. + """ + return self.value + + @classmethod + def from_charm(cls, charm: CharmBase) -> "BaseImage": + """Retrieve the base image tag from charm. + + Args: + charm: The charm instance. + + Returns: + The base image configuration of the charm. + """ + image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME, "jammy").lower().strip() + if image_name in LTS_IMAGE_VERSION_TAG_MAP: + return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) + return cls(image_name) + + +class InvalidImageConfigError(Exception): + """Represents an error with invalid image config.""" + + +@dataclasses.dataclass(frozen=True) +class ImageConfig: + """The charm configuration values related to image. + + Attributes: + arch: The underlying compute architecture, i.e. x86_64, amd64, arm64/aarch64. + base_image: The ubuntu base image to run the runner virtual machines on. + """ + + arch: Arch + base_image: BaseImage + + @classmethod + def from_charm(cls, charm: CharmBase) -> "ImageConfig": + """Initialize image config from charm instance. + + Args: + charm: The running charm instance. + + Raises: + InvalidImageConfigError: If an invalid image configuration value has been set. + + Returns: + Current charm image configuration state. + """ + try: + arch = _get_supported_arch() + except UnsupportedArchitectureError as exc: + raise InvalidImageConfigError( + f"Unsupported architecture {exc.arch}, please deploy on a supported architecture." + ) from exc + + try: + base_image = BaseImage.from_charm(charm) + except ValueError as exc: + raise InvalidImageConfigError( + ( + "Unsupported input option for base-image, please re-configure the base-image " + "option." + ) + ) from exc + + return cls(arch=arch, base_image=base_image) + + +def _parse_build_interval(charm: CharmBase) -> int: + """Parse build-interval charm configuration option. + + Args: + charm: The charm instance. + + Raises: + ValueError: If an invalid build interval is configured. + + Returns: + Build interval in hours. + """ + try: + build_interval = int(charm.config.get(BUILD_INTERVAL_CONFIG_NAME, 6)) + except ValueError as exc: + raise ValueError("An integer value for build-interval is expected.") from exc + if build_interval < 0 or build_interval > 24: + raise ValueError("Build interval must not be negative or greater than 24") + return build_interval + + +def _parse_revision_history_limit(charm: CharmBase) -> int: + """Parse revision-history-limit char configuration option. + + Args: + charm: The charm instance. + + Raises: + ValueError: If an invalid revision-history-limit is configured. + + Returns: + Number of revisions to keep before deletion. + """ + try: + revision_history = int(charm.config.get(REVISION_HISTORY_LIMIT_CONFIG_NAME, 5)) + except ValueError as exc: + raise ValueError("An integer value for revision history is expected.") from exc + if revision_history < 2 or revision_history > 99: + raise ValueError("Revision history must be greater than 1 and less than 100") + return revision_history + + +class InvalidCloudConfigError(Exception): + """Represents an error with openstack cloud config.""" + + +def _parse_openstack_clouds_config(charm: CharmBase) -> dict[str, Any]: + """Parse and validate openstack clouds yaml config value. + + Args: + charm: The charm instance. + + Raises: + InvalidCloudConfigError: if an invalid Openstack config value was set. + + Returns: + The openstack clouds yaml. + """ + openstack_clouds_yaml_str = charm.config.get(OPENSTACK_CLOUDS_YAML_CONFIG_NAME) + if not openstack_clouds_yaml_str: + raise InvalidCloudConfigError("No cloud config set") + + try: + openstack_clouds_yaml = yaml.safe_load(openstack_clouds_yaml_str) + except yaml.YAMLError as exc: + raise InvalidCloudConfigError( + f"Invalid {OPENSTACK_CLOUDS_YAML_CONFIG_NAME} config. Invalid yaml." + ) from exc + if (config_type := type(openstack_clouds_yaml)) is not dict: + raise InvalidCloudConfigError( + f"Invalid openstack config format, expected dict, got {config_type}" + ) + try: + clouds = list(openstack_clouds_yaml["clouds"].keys()) + except KeyError as exc: + raise InvalidCloudConfigError( + "Invalid openstack config. Not able to initialize openstack integration." + ) from exc + if not clouds: + raise InvalidCloudConfigError("No clouds found.") + + return openstack_clouds_yaml + + +class CharmConfigInvalidError(Exception): + """Raised when charm config is invalid. + + Attributes: + msg: Explanation of the error. + """ + + def __init__(self, msg: str): + """Initialize a new instance of the CharmConfigInvalidError exception. + + Args: + msg: Explanation of the error. + """ + self.msg = msg + + +@dataclasses.dataclass(frozen=True) +class CharmState: + """The charm state. + + Attributes: + build_interval: The interval in hours between each scheduled image builds. + cloud_config: The Openstack clouds.yaml passed as charm config. + image_config: The charm configuration values related to image. + proxy_config: The charm proxy configuration variables. + revision_history_limit: The number of image revisions to keep. + """ + + build_interval: int + cloud_config: dict[str, Any] + image_config: ImageConfig + proxy_config: ProxyConfig | None + revision_history_limit: int + + @classmethod + def from_charm(cls, charm: CharmBase) -> "CharmState": + """Initialize charm state from current charm instance. + + Args: + charm: The running charm instance. + + Raises: + CharmConfigInvalidError: If there was an invalid configuration on the charm. + + Returns: + Current charm state. + """ + try: + image_config = ImageConfig.from_charm(charm) + except InvalidImageConfigError as exc: + raise CharmConfigInvalidError(msg=str(exc)) from exc + + try: + build_interval = _parse_build_interval(charm) + except ValueError as exc: + raise CharmConfigInvalidError(msg=str(exc)) from exc + + try: + cloud_config = _parse_openstack_clouds_config(charm) + except InvalidCloudConfigError as exc: + raise CharmConfigInvalidError(msg=str(exc)) from exc + + try: + revision_history_limit = _parse_revision_history_limit(charm) + except ValueError as exc: + raise CharmConfigInvalidError(msg=str(exc)) from exc + + return cls( + build_interval=build_interval, + cloud_config=cloud_config, + image_config=image_config, + proxy_config=ProxyConfig.from_env(), + revision_history_limit=revision_history_limit, + ) diff --git a/src/github-runner-image-builder/utils.py b/src/github-runner-image-builder/utils.py new file mode 100644 index 0000000..daae1f4 --- /dev/null +++ b/src/github-runner-image-builder/utils.py @@ -0,0 +1,103 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Utilities used by the charm.""" + +import functools +import logging +import time +from typing import Callable, Optional, Type, TypeVar + +from typing_extensions import ParamSpec + +logger = logging.getLogger(__name__) + + +# Parameters of the function decorated with retry +ParamT = ParamSpec("ParamT") +# Return type of the function decorated with retry +ReturnT = TypeVar("ReturnT") + + +# This decorator has default arguments, one extra argument is not a problem. +def retry( # pylint: disable=too-many-arguments + exception: Type[Exception] = Exception, + tries: int = 1, + delay: float = 0, + max_delay: Optional[float] = None, + backoff: float = 1, + local_logger: logging.Logger = logger, +) -> Callable[[Callable[ParamT, ReturnT]], Callable[ParamT, ReturnT]]: + """Parameterize the decorator for adding retry to functions. + + Args: + exception: Exception type to be retried. + tries: Number of attempts at retry. + delay: Time in seconds to wait between retry. + max_delay: Max time in seconds to wait between retry. + backoff: Factor to increase the delay by each retry. + local_logger: Logger for logging. + + Returns: + The function decorator for retry. + """ + + def retry_decorator( + func: Callable[ParamT, ReturnT], + ) -> Callable[ParamT, ReturnT]: + """Decorate function with retry. + + Args: + func: The function to decorate. + + Returns: + The resulting function with retry added. + """ + + @functools.wraps(func) + def fn_with_retry(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT: + """Wrap the function with retries. + + Args: + args: The placeholder for decorated function's positional arguments. + kwargs: The placeholder for decorated function's key word arguments. + + Raises: + RuntimeError: Should be unreachable. + + Returns: + Original return type of the decorated function. + """ + remain_tries, current_delay = tries, delay + + for _ in range(tries): + try: + print("Trying") + return func(*args, **kwargs) + # Error caught is set by the input of the function. + except exception as err: # pylint: disable=broad-exception-caught + remain_tries -= 1 + + if remain_tries == 0: + if local_logger is not None: + local_logger.exception("Retry limit of %s exceed: %s", tries, err) + raise + + if local_logger is not None: + local_logger.warning( + "Retrying error in %s seconds: %s", current_delay, err + ) + local_logger.debug("Error to be retried:", stack_info=True) + + time.sleep(current_delay) + + current_delay *= backoff + + if max_delay is not None: + current_delay = min(current_delay, max_delay) + + raise RuntimeError("Unreachable code of retry logic.") # pragma: nocover + + return fn_with_retry + + return retry_decorator From faa8ba26f52b903d26a6fbac01a33e809e6fa0c9 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 29 May 2024 03:34:39 +0000 Subject: [PATCH 02/63] feat: initial application --- .github/ISSUE_TEMPLATE/bug_report.yml | 55 ++ .../ISSUE_TEMPLATE/enhancement_proposal.yml | 17 + .github/pull_request_template.md | 26 + .github/workflows/comment.yaml | 12 + .github/workflows/integration_test.yaml | 28 + .github/workflows/issues.yaml | 11 + .github/workflows/test.yaml | 11 + .gitignore | 9 + .licenserc.yaml | 30 + .woke.yaml | 3 + README.md | 0 generate-src-docs.sh | 6 + pyproject.toml | 67 +++ requirements.txt | 3 + snap/snapcraft.yaml | 34 -- src/github-runner-image-builder/state.py | 335 ----------- src/github_runner_image_builder/__init__.py | 4 + src/github_runner_image_builder/__main__.py | 11 + .../builder.py | 512 ++++++++-------- .../chroot.py | 4 +- src/github_runner_image_builder/cli.py | 199 ++++++ src/github_runner_image_builder/config.py | 125 ++++ src/github_runner_image_builder/errors.py | 89 +++ src/github_runner_image_builder/upload.py | 159 +++++ .../utils.py | 2 +- tests/conftest.py | 57 ++ tests/integration/__init__.py | 4 + tests/integration/conftest.py | 155 +++++ tests/integration/data/clouds.yaml.tmpl | 10 + tests/integration/helpers.py | 204 +++++++ tests/integration/requirements.txt | 1 + tests/integration/test_image.py | 137 +++++ tests/integration/testdata/metadata.yaml.tmpl | 13 + .../testdata/templates/hostname.tpl | 1 + .../unit}/__init__.py | 1 + tests/unit/factories.py | 37 ++ tests/unit/requirements.txt | 1 + tests/unit/test_builder.py | 565 ++++++++++++++++++ tests/unit/test_chroot.py | 124 ++++ tests/unit/test_cli.py | 230 +++++++ tests/unit/test_config.py | 96 +++ tests/unit/test_upload.py | 227 +++++++ tests/unit/test_utils.py | 78 +++ tox.ini | 119 ++++ 44 files changed, 3176 insertions(+), 636 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/enhancement_proposal.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/comment.yaml create mode 100644 .github/workflows/integration_test.yaml create mode 100644 .github/workflows/issues.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .gitignore create mode 100644 .licenserc.yaml create mode 100644 .woke.yaml create mode 100644 README.md create mode 100644 generate-src-docs.sh create mode 100644 requirements.txt delete mode 100644 snap/snapcraft.yaml delete mode 100644 src/github-runner-image-builder/state.py create mode 100644 src/github_runner_image_builder/__init__.py create mode 100644 src/github_runner_image_builder/__main__.py rename src/{github-runner-image-builder => github_runner_image_builder}/builder.py (55%) rename src/{github-runner-image-builder => github_runner_image_builder}/chroot.py (98%) create mode 100644 src/github_runner_image_builder/cli.py create mode 100644 src/github_runner_image_builder/config.py create mode 100644 src/github_runner_image_builder/errors.py create mode 100644 src/github_runner_image_builder/upload.py rename src/{github-runner-image-builder => github_runner_image_builder}/utils.py (98%) create mode 100644 tests/conftest.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/data/clouds.yaml.tmpl create mode 100644 tests/integration/helpers.py create mode 100644 tests/integration/requirements.txt create mode 100644 tests/integration/test_image.py create mode 100644 tests/integration/testdata/metadata.yaml.tmpl create mode 100644 tests/integration/testdata/templates/hostname.tpl rename {src/github-runner-image-builder => tests/unit}/__init__.py (75%) create mode 100644 tests/unit/factories.py create mode 100644 tests/unit/requirements.txt create mode 100644 tests/unit/test_builder.py create mode 100644 tests/unit/test_chroot.py create mode 100644 tests/unit/test_cli.py create mode 100644 tests/unit/test_config.py create mode 100644 tests/unit/test_upload.py create mode 100644 tests/unit/test_utils.py create mode 100644 tox.ini diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..d6b22e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,55 @@ +name: Bug Report +description: File a bug report +labels: ["Type: Bug", "Status: Triage"] +body: + - type: markdown + attributes: + value: > + Thanks for taking the time to fill out this bug report! Before submitting your issue, please make + sure you are using the latest version of the app. If not, please switch to this image prior to + posting your report to make sure it's not already solved. + - type: textarea + id: bug-description + attributes: + label: Bug Description + description: > + If applicable, add screenshots to help explain the problem you are facing. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: To Reproduce + description: > + Please provide a step-by-step instruction of how to reproduce the behavior. + placeholder: | + 1. pipx install . + 2. github-runner-image-builder install + 3. github-runner-image-builder build + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: > + We need to know a bit more about the context in which you run the app. + - Are you running the application locally, on lxd, in multipass or on some other platform? + - Version of any applicable components, like the pipx/pip lxd, and/or multipass. + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: > + Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + Fetch the logs using `github-runner-image-builder &> logs.txt` + render: shell + validations: + required: true + - type: textarea + id: additional-context + attributes: + label: Additional context + diff --git a/.github/ISSUE_TEMPLATE/enhancement_proposal.yml b/.github/ISSUE_TEMPLATE/enhancement_proposal.yml new file mode 100644 index 0000000..b2348b9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement_proposal.yml @@ -0,0 +1,17 @@ +name: Enhancement Proposal +description: File an enhancement proposal +labels: ["Type: Enhancement", "Status: Triage"] +body: + - type: markdown + attributes: + value: > + Thanks for taking the time to fill out this enhancement proposal! Before submitting your issue, please make + sure there isn't already a prior issue concerning this. If there is, please join that discussion instead. + - type: textarea + id: enhancement-proposal + attributes: + label: Enhancement Proposal + description: > + Describe the enhancement you would like to see in as much detail as needed. + validations: + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..e3d60f2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,26 @@ +Applicable spec: + +### Overview + + + +### Rationale + + + + +### Module Changes + + + +### Library/Dependency Changes + + + +### Checklist + +- [ ] The [contributing guide](https://github.com/canonical/is-charms-contributing-guide) was applied +- [ ] The documentation is generated using `src-docs` +- [ ] The PR is tagged with appropriate label (`urgent`, `trivial`, `complex`) + + diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml new file mode 100644 index 0000000..26ac226 --- /dev/null +++ b/.github/workflows/comment.yaml @@ -0,0 +1,12 @@ +name: Comment on the pull request + +on: + workflow_run: + workflows: ["Tests"] + types: + - completed + +jobs: + comment-on-pr: + uses: canonical/operator-workflows/.github/workflows/comment.yaml@main + secrets: inherit diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml new file mode 100644 index 0000000..58069e5 --- /dev/null +++ b/.github/workflows/integration_test.yaml @@ -0,0 +1,28 @@ +name: Integration tests + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + integration-tests: + name: Integration test + runs-on: [self-hosted, stg-private-endpoint] + strategy: + matrix: + image: [jammy, noble] + steps: + - uses: actions/checkout@v3 + - uses: canonical/setup-lxd@v0.1.1 + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.py }} + # need to run in sudo mode due to chroot + - name: Install tox + run: sudo python -m pip install tox-gh + - name: Run integration tests + run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} diff --git a/.github/workflows/issues.yaml b/.github/workflows/issues.yaml new file mode 100644 index 0000000..138fe82 --- /dev/null +++ b/.github/workflows/issues.yaml @@ -0,0 +1,11 @@ +name: Sync issues to Jira + +on: + issues: + # available via github.event.action + types: [opened, reopened, closed] + +jobs: + issues-to-jira: + uses: canonical/operator-workflows/.github/workflows/jira.yaml@main + secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..138c4a4 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,11 @@ +name: Tests + +on: + pull_request: + +jobs: + unit-tests: + uses: canonical/operator-workflows/.github/workflows/test.yaml@main + secrets: inherit + with: + self-hosted-runner: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3a09791 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# development cofig files +.vscode +.tox +**/__pycache__ +.coverage +# build artefacts +build +*.img +**/*.egg-info diff --git a/.licenserc.yaml b/.licenserc.yaml new file mode 100644 index 0000000..178de89 --- /dev/null +++ b/.licenserc.yaml @@ -0,0 +1,30 @@ +header: + license: + spdx-id: Apache-2.0 + copyright-owner: Canonical Ltd. + content: | + Copyright [year] [owner] + See LICENSE file for licensing details. + paths: + - '**' + paths-ignore: + - '.github/**' + - '**/*.j2' + - '**/*.json' + - '**/*.md' + - '**/*.txt' + - '.codespellignore' + - '.copier-answers.yml' + - '.flake8' + - '.jujuignore' + - '.gitignore' + - '.licenserc.yaml' + - 'CODEOWNERS' + - 'icon.svg' + - 'LICENSE' + - '.pylintrc' + - '.woke.yaml' + - 'lib/**' + - 'tests/integration/testdata/**' + - 'tests/integration/data/**' + comment: on-failure diff --git a/.woke.yaml b/.woke.yaml new file mode 100644 index 0000000..03831c9 --- /dev/null +++ b/.woke.yaml @@ -0,0 +1,3 @@ +rules: + # Ignore blacklist - we are using it to ignore bandit check + - name: blacklist diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/generate-src-docs.sh b/generate-src-docs.sh new file mode 100644 index 0000000..d13066a --- /dev/null +++ b/generate-src-docs.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +lazydocs --no-watermark --output-path src-docs src/* diff --git a/pyproject.toml b/pyproject.toml index b2abfe8..87cfdd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + [project] name = "github-runner-image-builder" version = "0.0.1" @@ -10,7 +13,71 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] +dynamic = ["dependencies"] +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } [project.urls] Homepage = "https://github.com/canonical/github-runner-image-builder-snap" Issues = "https://github.com/canonical/github-runner-image-builder-snap/issues" + +[project.scripts] +github-runner-image-builder = "github_runner_image_builder.cli:main" + +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +[tool.bandit] +exclude_dirs = ["/venv/"] +[tool.bandit.assert_used] +skips = ["*/*test.py", "*/test_*.py", "*tests/*.py"] + +# Testing tools configuration +[tool.coverage.run] +branch = true + +[tool.coverage.report] +fail_under = 100 +show_missing = true + +[tool.pytest.ini_options] +minversion = "6.0" +log_cli_level = "INFO" + +# Formatting tools configuration +[tool.black] +line-length = 99 +target-version = ["py310"] + +[tool.isort] +line_length = 99 +profile = "black" + +# Linting tools configuration +[tool.flake8] +max-line-length = 99 +max-doc-length = 99 +max-complexity = 10 +exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] +select = ["E", "W", "F", "C", "N", "R", "D", "H"] +# Ignore W503 because using black creates errors with this +# Ignore D107 Missing docstring in __init__ +ignore = ["W503", "D107"] +# D100, D101, D102, D103, D104, DCO020, DCO030: Ignore docstring style issues in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212,DCO020,DCO030"] +docstring-convention = "google" +# Check for properly formatted copyright header in each file +copyright-check = "True" +copyright-author = "Canonical Ltd." +copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" + +[tool.mypy] +check_untyped_defs = true +disallow_untyped_defs = true +explicit_package_bases = true +ignore_missing_imports = true +namespace_packages = true + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_untyped_defs = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..799c087 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +typing-extensions==4.11.0 +pyyaml==6.0.1 +openstacksdk==3.1.0 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml deleted file mode 100644 index 7cd430c..0000000 --- a/snap/snapcraft.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: github-runner-image-builder -base: core22 # the base snap is the execution environment for this snap -version: "0.1" # just for humans, typically '1.2+git' or '1.3.2' -summary: The snap for building images for github-runners -description: | - Github-runner-image-builder is a snap for building ubuntu images for - github-runner charm (https://github.com/canonical/github-runner-operator/) - used by the github-runner-image-builder charm. It periodically builds - new images according to the configuration. - -license: Apache-2.0 -grade: stable -confinement: strict -architectures: - - build-on: amd64 - build-for: amd64 - - build-on: arm64 - build-for: arm64 - -apps: - github-runner-image-builder: - command: bin/github-runner-image-builder - daemon: simple - -parts: - github-runner-image-builder: - plugin: python - source: . - stage-packages: - - "qemu-utils" - - "libguestfs-tools" - -hooks: - configure: {} diff --git a/src/github-runner-image-builder/state.py b/src/github-runner-image-builder/state.py deleted file mode 100644 index 0397163..0000000 --- a/src/github-runner-image-builder/state.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Module for interacting with charm state and configurations.""" - -import dataclasses -import logging -import os -import platform -from enum import Enum -from typing import Any, Optional - -import yaml -from ops import CharmBase - -logger = logging.getLogger(__name__) - -BASE_IMAGE_CONFIG_NAME = "base-image" -BUILD_INTERVAL_CONFIG_NAME = "build-interval" -OPENSTACK_CLOUDS_YAML_CONFIG_NAME = "experimental-openstack-clouds-yaml" -REVISION_HISTORY_LIMIT_CONFIG_NAME = "revision-history-limit" - - -class Arch(str, Enum): - """Supported system architectures. - - Attributes: - ARM64: Represents an ARM64 system architecture. - X64: Represents an X64/AMD64 system architecture. - """ - - def __str__(self) -> str: - """Interpolate to string value. - - Returns: - The enum string value. - """ - return self.value - - ARM64 = "arm64" - X64 = "x64" - - -class UnsupportedArchitectureError(Exception): - """Raised when given machine charm architecture is unsupported. - - Attributes: - arch: The current machine architecture. - """ - - def __str__(self) -> str: - """Represent the error in string format. - - Returns: - The error in string format. - """ - return f"UnsupportedArchitectureError: {self.arch}" - - def __init__(self, arch: str) -> None: - """Initialize a new instance of the CharmConfigInvalidError exception. - - Args: - arch: The current machine architecture. - """ - self.arch = arch - - -ARCHITECTURES_ARM64 = {"aarch64", "arm64"} -ARCHITECTURES_X86 = {"x86_64"} - - -def _get_supported_arch() -> Arch: - """Get current machine architecture. - - Raises: - UnsupportedArchitectureError: if the current architecture is unsupported. - - Returns: - Arch: Current machine architecture. - """ - arch = platform.machine() - match arch: - case arch if arch in ARCHITECTURES_ARM64: - return Arch.ARM64 - case arch if arch in ARCHITECTURES_X86: - return Arch.X64 - case _: - raise UnsupportedArchitectureError(arch=arch) - - -LTS_IMAGE_VERSION_TAG_MAP = {"22.04": "jammy", "24.04": "noble"} - - -class BaseImage(str, Enum): - """The ubuntu OS base image to build and deploy runners on. - - Attributes: - JAMMY: The jammy ubuntu LTS image. - NOBLE: The noble ubuntu LTS image. - """ - - JAMMY = "jammy" - NOBLE = "noble" - - def __str__(self) -> str: - """Interpolate to string value. - - Returns: - The enum string value. - """ - return self.value - - @classmethod - def from_charm(cls, charm: CharmBase) -> "BaseImage": - """Retrieve the base image tag from charm. - - Args: - charm: The charm instance. - - Returns: - The base image configuration of the charm. - """ - image_name = charm.config.get(BASE_IMAGE_CONFIG_NAME, "jammy").lower().strip() - if image_name in LTS_IMAGE_VERSION_TAG_MAP: - return cls(LTS_IMAGE_VERSION_TAG_MAP[image_name]) - return cls(image_name) - - -class InvalidImageConfigError(Exception): - """Represents an error with invalid image config.""" - - -@dataclasses.dataclass(frozen=True) -class ImageConfig: - """The charm configuration values related to image. - - Attributes: - arch: The underlying compute architecture, i.e. x86_64, amd64, arm64/aarch64. - base_image: The ubuntu base image to run the runner virtual machines on. - """ - - arch: Arch - base_image: BaseImage - - @classmethod - def from_charm(cls, charm: CharmBase) -> "ImageConfig": - """Initialize image config from charm instance. - - Args: - charm: The running charm instance. - - Raises: - InvalidImageConfigError: If an invalid image configuration value has been set. - - Returns: - Current charm image configuration state. - """ - try: - arch = _get_supported_arch() - except UnsupportedArchitectureError as exc: - raise InvalidImageConfigError( - f"Unsupported architecture {exc.arch}, please deploy on a supported architecture." - ) from exc - - try: - base_image = BaseImage.from_charm(charm) - except ValueError as exc: - raise InvalidImageConfigError( - ( - "Unsupported input option for base-image, please re-configure the base-image " - "option." - ) - ) from exc - - return cls(arch=arch, base_image=base_image) - - -def _parse_build_interval(charm: CharmBase) -> int: - """Parse build-interval charm configuration option. - - Args: - charm: The charm instance. - - Raises: - ValueError: If an invalid build interval is configured. - - Returns: - Build interval in hours. - """ - try: - build_interval = int(charm.config.get(BUILD_INTERVAL_CONFIG_NAME, 6)) - except ValueError as exc: - raise ValueError("An integer value for build-interval is expected.") from exc - if build_interval < 0 or build_interval > 24: - raise ValueError("Build interval must not be negative or greater than 24") - return build_interval - - -def _parse_revision_history_limit(charm: CharmBase) -> int: - """Parse revision-history-limit char configuration option. - - Args: - charm: The charm instance. - - Raises: - ValueError: If an invalid revision-history-limit is configured. - - Returns: - Number of revisions to keep before deletion. - """ - try: - revision_history = int(charm.config.get(REVISION_HISTORY_LIMIT_CONFIG_NAME, 5)) - except ValueError as exc: - raise ValueError("An integer value for revision history is expected.") from exc - if revision_history < 2 or revision_history > 99: - raise ValueError("Revision history must be greater than 1 and less than 100") - return revision_history - - -class InvalidCloudConfigError(Exception): - """Represents an error with openstack cloud config.""" - - -def _parse_openstack_clouds_config(charm: CharmBase) -> dict[str, Any]: - """Parse and validate openstack clouds yaml config value. - - Args: - charm: The charm instance. - - Raises: - InvalidCloudConfigError: if an invalid Openstack config value was set. - - Returns: - The openstack clouds yaml. - """ - openstack_clouds_yaml_str = charm.config.get(OPENSTACK_CLOUDS_YAML_CONFIG_NAME) - if not openstack_clouds_yaml_str: - raise InvalidCloudConfigError("No cloud config set") - - try: - openstack_clouds_yaml = yaml.safe_load(openstack_clouds_yaml_str) - except yaml.YAMLError as exc: - raise InvalidCloudConfigError( - f"Invalid {OPENSTACK_CLOUDS_YAML_CONFIG_NAME} config. Invalid yaml." - ) from exc - if (config_type := type(openstack_clouds_yaml)) is not dict: - raise InvalidCloudConfigError( - f"Invalid openstack config format, expected dict, got {config_type}" - ) - try: - clouds = list(openstack_clouds_yaml["clouds"].keys()) - except KeyError as exc: - raise InvalidCloudConfigError( - "Invalid openstack config. Not able to initialize openstack integration." - ) from exc - if not clouds: - raise InvalidCloudConfigError("No clouds found.") - - return openstack_clouds_yaml - - -class CharmConfigInvalidError(Exception): - """Raised when charm config is invalid. - - Attributes: - msg: Explanation of the error. - """ - - def __init__(self, msg: str): - """Initialize a new instance of the CharmConfigInvalidError exception. - - Args: - msg: Explanation of the error. - """ - self.msg = msg - - -@dataclasses.dataclass(frozen=True) -class CharmState: - """The charm state. - - Attributes: - build_interval: The interval in hours between each scheduled image builds. - cloud_config: The Openstack clouds.yaml passed as charm config. - image_config: The charm configuration values related to image. - proxy_config: The charm proxy configuration variables. - revision_history_limit: The number of image revisions to keep. - """ - - build_interval: int - cloud_config: dict[str, Any] - image_config: ImageConfig - proxy_config: ProxyConfig | None - revision_history_limit: int - - @classmethod - def from_charm(cls, charm: CharmBase) -> "CharmState": - """Initialize charm state from current charm instance. - - Args: - charm: The running charm instance. - - Raises: - CharmConfigInvalidError: If there was an invalid configuration on the charm. - - Returns: - Current charm state. - """ - try: - image_config = ImageConfig.from_charm(charm) - except InvalidImageConfigError as exc: - raise CharmConfigInvalidError(msg=str(exc)) from exc - - try: - build_interval = _parse_build_interval(charm) - except ValueError as exc: - raise CharmConfigInvalidError(msg=str(exc)) from exc - - try: - cloud_config = _parse_openstack_clouds_config(charm) - except InvalidCloudConfigError as exc: - raise CharmConfigInvalidError(msg=str(exc)) from exc - - try: - revision_history_limit = _parse_revision_history_limit(charm) - except ValueError as exc: - raise CharmConfigInvalidError(msg=str(exc)) from exc - - return cls( - build_interval=build_interval, - cloud_config=cloud_config, - image_config=image_config, - proxy_config=ProxyConfig.from_env(), - revision_history_limit=revision_history_limit, - ) diff --git a/src/github_runner_image_builder/__init__.py b/src/github_runner_image_builder/__init__.py new file mode 100644 index 0000000..d9ae4a9 --- /dev/null +++ b/src/github_runner_image_builder/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Github runner image builder module.""" diff --git a/src/github_runner_image_builder/__main__.py b/src/github_runner_image_builder/__main__.py new file mode 100644 index 0000000..9016dde --- /dev/null +++ b/src/github_runner_image_builder/__main__.py @@ -0,0 +1,11 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Main entrypoint for github-runner-image-builder.""" + +import sys + +from github_runner_image_builder.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/github-runner-image-builder/builder.py b/src/github_runner_image_builder/builder.py similarity index 55% rename from src/github-runner-image-builder/builder.py rename to src/github_runner_image_builder/builder.py index 697ff60..5d500bc 100644 --- a/src/github-runner-image-builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -4,27 +4,120 @@ """Module for interacting with qemu image builder.""" import dataclasses -import hashlib import logging import os import shutil # Ignore B404:blacklist since all subprocesses are run with predefined executables. import subprocess # nosec +import sys import urllib.error import urllib.request +from contextlib import redirect_stdout from pathlib import Path from typing import Literal -from chroot import ChrootBaseError, ChrootContextManager -from state import Arch, BaseImage -from utils import retry +from github_runner_image_builder.chroot import ChrootBaseError, ChrootContextManager +from github_runner_image_builder.config import IMAGE_OUTPUT_PATH, Arch, BaseImage +from github_runner_image_builder.errors import ( + BuilderSetupError, + BuildImageError, + CleanBuildStateError, + CloudImageDownloadError, + DependencyInstallError, + ExternalPackageInstallError, + ImageBuilderBaseError, + ImageCompressError, + ImageMountError, + ImageResizeError, + NetworkBlockDeviceError, + ResizePartitionError, + SystemUserConfigurationError, + UnattendedUpgradeDisableError, + UnsupportedArchitectureError, + YQBuildError, +) +from github_runner_image_builder.utils import retry logger = logging.getLogger(__name__) +SupportedCloudImageArch = Literal["amd64", "arm64"] -class NetworkBlockDeviceError(Exception): - """Represents an error while enabling network block device.""" +APT_DEPENDENCIES = [ + "qemu-utils", # used for qemu utilities tools to build and resize image + "libguestfs-tools", # used to modify VM images. + "cloud-utils", # used for growpart. + "golang-go", # used to build yq from source. +] +SNAP_GO = "go" + +# Constants for mounting images +IMAGE_MOUNT_DIR = Path("/mnt/ubuntu-image/") +NETWORK_BLOCK_DEVICE_PATH = Path("/dev/nbd0") +NETWORK_BLOCK_DEVICE_PARTITION_PATH = Path("/dev/nbd0p1") + +# Constants for building image +# This amount is the smallest increase that caters for the installations within this image. +RESIZE_AMOUNT = "+1.5G" +MOUNTED_RESOLV_CONF_PATH = IMAGE_MOUNT_DIR / "etc/resolv.conf" +HOST_RESOLV_CONF_PATH = Path("/etc/resolv.conf") + +# Constants for chroot environment Python symmlinks +DEFAULT_PYTHON_PATH = Path("/usr/bin/python3") +SYM_LINK_PYTHON_PATH = Path("/usr/bin/python") + +# Constants for disabling automatic apt updates +APT_TIMER = "apt-daily.timer" +APT_SVC = "apt-daily.service" +APT_UPGRADE_TIMER = "apt-daily-upgrade.timer" +APT_UPGRAD_SVC = "apt-daily-upgrade.service" + +# Constants for managing users and groups +UBUNTU_USER = "ubuntu" +DOCKER_GROUP = "docker" +MICROK8S_GROUP = "microk8s" +LXD_GROUP = "lxd" +UBUNTU_HOME = Path("/home/ubuntu") + +# Constants for packages in the image +YQ_REPOSITORY_URL = "https://github.com/mikefarah/yq.git" +YQ_REPOSITORY_PATH = Path("yq_source") +HOST_YQ_BIN_PATH = Path("/usr/bin/yq") +MOUNTED_YQ_BIN_PATH = IMAGE_MOUNT_DIR / "usr/bin/yq" +IMAGE_DEFAULT_APT_PACKAGES = [ + "docker.io", + "npm", + "python3-pip", + "shellcheck", + "jq", + "wget", + "unzip", + "gh", +] + + +def _install_dependencies() -> None: + """Install required dependencies to run qemu image build. + + Raises: + DependencyInstallError: If there was an error installing apt packages. + """ + try: + subprocess.run( + ["/usr/bin/apt-get", "update", "-y"], check=True, timeout=30 * 60 + ) # nosec: B603 + subprocess.run( + ["/usr/bin/apt-get", "install", "-y", "--no-install-recommends", *APT_DEPENDENCIES], + check=True, + timeout=30 * 60, + ) # nosec: B603 + subprocess.run( + ["/usr/bin/snap", "install", SNAP_GO, "--classic"], + check=True, + timeout=30 * 60, + ) # nosec: B603 + except subprocess.CalledProcessError as exc: + raise DependencyInstallError from exc def _enable_nbd() -> None: @@ -39,10 +132,6 @@ def _enable_nbd() -> None: raise NetworkBlockDeviceError from exc -class BuilderSetupError(Exception): - """Represents an error while setting up host machine as builder.""" - - def setup_builder() -> None: """Configure the host machine to build images. @@ -50,30 +139,12 @@ def setup_builder() -> None: BuilderSetupError: If there was an error setting up the host device for building images. """ try: + _install_dependencies() _enable_nbd() - except NetworkBlockDeviceError as exc: + except ImageBuilderBaseError as exc: raise BuilderSetupError from exc -class UnsupportedArchitectureError(Exception): - """Raised when given machine charm architecture is unsupported. - - Attributes: - arch: The current machine architecture. - """ - - def __init__(self, arch: str) -> None: - """Initialize a new instance of the CharmConfigInvalidError exception. - - Args: - arch: The current machine architecture. - """ - self.arch = arch - - -SupportedCloudImageArch = Literal["amd64", "arm64"] - - def _get_supported_runner_arch(arch: Arch) -> SupportedCloudImageArch: """Validate and return supported runner architecture. @@ -98,58 +169,49 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedCloudImageArch: case Arch.ARM64: return "arm64" case _: - raise UnsupportedArchitectureError(arch) - - -IMAGE_MOUNT_DIR = Path("/mnt/ubuntu-image/") -NETWORK_BLOCK_DEVICE_PATH = Path("/dev/nbd0") -NETWORK_BLOCK_DEVICE_PARTITION_PATH = Path("/dev/nbd0p1") + raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") def _clean_build_state() -> None: - """Remove any artefacts left by previous build.""" + """Remove any artefacts left by previous build. + + Raises: + CleanBuildStateError: if there was an error cleaning up the build state. + """ # The commands will fail if artefacts do not exist and hence there is no need to check the # output of subprocess runs. IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) - subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False - ) # nosec: B603 - subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], timeout=30, check=False - ) # nosec: B603 - subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], timeout=30, check=False - ) # nosec: B603 - subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False - ) # nosec: B603 - subprocess.run( - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False - ) # nosec: B603 - subprocess.run( # nosec: B603 - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False - ) - subprocess.run( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], - timeout=30, - check=False, - ) - subprocess.run( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], - timeout=30, - check=False, - ) - - -CLOUD_IMAGE_URL_TMPL = ( - "https://cloud-images.ubuntu.com/{BASE_IMAGE}/current/" - "{BASE_IMAGE}-server-cloudimg-{BIN_ARCH}.img" -) -CLOUD_IMAGE_FILE_NAME = "{BASE_IMAGE}-server-cloudimg-{BIN_ARCH}.img" - - -class CloudImageDownloadError(Exception): - """Represents an error downloading cloud image.""" + try: + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False + ) # nosec: B603 + subprocess.run( + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False + ) # nosec: B603 + subprocess.run( # nosec: B603 + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False + ) + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], + timeout=30, + check=False, + ) + subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + timeout=30, + check=False, + ) + except subprocess.SubprocessError as exc: + raise CleanBuildStateError from exc def _download_cloud_image(arch: Arch, base_image: BaseImage) -> Path: @@ -172,19 +234,19 @@ def _download_cloud_image(arch: Arch, base_image: BaseImage) -> Path: try: # The ubuntu-cloud-images is a trusted source - image_path, _ = urllib.request.urlretrieve( # nosec: B310 - CLOUD_IMAGE_URL_TMPL.format(BASE_IMAGE=base_image.value, BIN_ARCH=bin_arch), - CLOUD_IMAGE_FILE_NAME.format(BASE_IMAGE=base_image.value, BIN_ARCH=bin_arch), + image_path = f"{base_image.value}-server-cloudimg-{bin_arch}.img" + urllib.request.urlretrieve( # nosec: B310 + ( + f"https://cloud-images.ubuntu.com/{base_image.value}/current/{base_image.value}" + f"-server-cloudimg-{bin_arch}.img" + ), + image_path, ) return Path(image_path) - except urllib.error.ContentTooShortError as exc: + except urllib.error.URLError as exc: raise CloudImageDownloadError from exc -class ImageResizeError(Exception): - """Represents an error while resizing the image.""" - - def _resize_cloud_img(cloud_image_path: Path) -> None: """Resize cloud image to allow space for dependency installations. @@ -196,16 +258,14 @@ def _resize_cloud_img(cloud_image_path: Path) -> None: """ try: subprocess.run( # nosec: B603 - ["/usr/bin/qemu-img", "resize", str(cloud_image_path), "+1.5G"], check=True, timeout=60 + ["/usr/bin/qemu-img", "resize", str(cloud_image_path), RESIZE_AMOUNT], + check=True, + timeout=60, ) except subprocess.CalledProcessError as exc: raise ImageResizeError from exc -class ImageMountError(Exception): - """Represents an error while mounting the image to network block device.""" - - @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _mount_nbd_partition() -> None: """Mount the network block device partition.""" @@ -242,20 +302,12 @@ def _mount_image_to_network_block_device(cloud_image_path: Path) -> None: raise ImageMountError from exc -MOUNTED_RESOLV_CONF_PATH = IMAGE_MOUNT_DIR / "etc/resolv.conf" -HOST_RESOLV_CONF_PATH = Path("/etc/resolv.conf") - - def _replace_mounted_resolv_conf() -> None: """Replace resolv.conf to host resolv.conf to allow networking.""" MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) -class ResizePartitionError(Exception): - """Represents an error while resizing network block device partitions.""" - - def _resize_mount_partitions() -> None: """Resize the block partition to fill available space. @@ -264,21 +316,44 @@ def _resize_mount_partitions() -> None: """ try: subprocess.run( # nosec: B603 - ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], - check=True, - timeout=60, + ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], check=True, timeout=10 * 60 ) subprocess.run( # nosec: B603 ["/usr/sbin/resize2fs", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], check=True, - timeout=60, + timeout=10 * 60, ) except subprocess.CalledProcessError as exc: raise ResizePartitionError from exc -DEFAULT_PYTHON_PATH = Path("/usr/bin/python3") -SYM_LINK_PYTHON_PATH = Path("/usr/bin/python") +def _install_yq() -> None: + """Build and install yq from source. + + Raises: + YQBuildError: If there was an error building yq from source. + """ + try: + if not YQ_REPOSITORY_PATH.exists(): + subprocess.run( # nosec: B603 + ["/usr/bin/git", "clone", str(YQ_REPOSITORY_URL), str(YQ_REPOSITORY_PATH)], + check=True, + timeout=60 * 10, + ) + else: + subprocess.run( # nosec: B603 + ["/usr/bin/git", "-C", str(YQ_REPOSITORY_PATH), "pull"], + check=True, + timeout=60 * 10, + ) + subprocess.run( # nosec: B603 + ["/snap/bin/go", "build", "-C", str(YQ_REPOSITORY_PATH), "-o", str(HOST_YQ_BIN_PATH)], + check=True, + timeout=20 * 60, + ) + shutil.copy(HOST_YQ_BIN_PATH, MOUNTED_YQ_BIN_PATH) + except subprocess.CalledProcessError as exc: + raise YQBuildError from exc def _create_python_symlinks() -> None: @@ -286,16 +361,6 @@ def _create_python_symlinks() -> None: os.symlink(DEFAULT_PYTHON_PATH, SYM_LINK_PYTHON_PATH) -APT_TIMER = "apt-daily.timer" -APT_SVC = "apt-daily.service" -APT_UPGRADE_TIMER = "apt-daily-upgrade.timer" -APT_UPGRAD_SVC = "apt-daily-upgrade.service" - - -class UnattendedUpgradeDisableError(Exception): - """Represents an error while disabling unattended-upgrade related services.""" - - def _disable_unattended_upgrades() -> None: """Disable unatteneded upgrades to prevent apt locks. @@ -334,16 +399,6 @@ def _disable_unattended_upgrades() -> None: raise UnattendedUpgradeDisableError from exc -class SystemUserConfigurationError(Exception): - """Represents an error while adding user to chroot env.""" - - -UBUNTU_USER = "ubuntu" -DOCKER_GROUP = "docker" -MICROK8S_GROUP = "microk8s" -UBUNUT_HOME_PATH = Path("/home/ubuntu") - - def _configure_system_users() -> None: """Configure system users. @@ -351,11 +406,13 @@ def _configure_system_users() -> None: SystemUserConfigurationError: If there was an error configuring ubuntu user. """ try: - with (UBUNUT_HOME_PATH / ".profile").open("a") as profile_file: - profile_file.write("PATH=$PATH:/home/ubuntu/.local/bin\n") subprocess.run( # nosec: B603 ["/usr/sbin/useradd", "-m", UBUNTU_USER], check=True, timeout=30 ) + with (UBUNTU_HOME / ".profile").open("a") as profile_file: + profile_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") + with (UBUNTU_HOME / ".bashrc").open("a") as bashrc_file: + bashrc_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") subprocess.run( # nosec: B603 ["/usr/sbin/groupadd", MICROK8S_GROUP], check=True, timeout=30 ) @@ -365,6 +422,9 @@ def _configure_system_users() -> None: subprocess.run( # nosec: B603 ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], check=True, timeout=30 ) + subprocess.run( # nosec: B603 + ["/usr/sbin/usermod", "-aG", LXD_GROUP, UBUNTU_USER], check=True, timeout=30 + ) subprocess.run( # nosec: B603 ["/usr/bin/chmod", "777", "/usr/local/bin"], check=True, timeout=30 ) @@ -372,92 +432,28 @@ def _configure_system_users() -> None: raise SystemUserConfigurationError from exc -YQ_DOWNLOAD_URL_TMPL = ( - "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_{BIN_ARCH}" -) -YQ_BINARY_CHECKSUM_URL = "https://github.com/mikefarah/yq/releases/latest/download/checksums" -YQ_CHECKSUM_HASHES_ORDER_URL = ( - "https://github.com/mikefarah/yq/releases/latest/download/checksums_hashes_order" -) -YQ_EXTRACT_CHECKSUM_SCRIPT_URL = ( - "https://github.com/mikefarah/yq/releases/latest/download/extract-checksum.sh" -) - - -class ExternalPackageInstallError(Exception): - """Represents an error installilng external packages.""" - - -def _validate_checksum(file: Path, expected_checksum: str) -> bool: - """Validate the checksum of a given file. - - Args: - file: The file to calculate checksum for. - expected_checksum: The expected file checksum. - - Returns: - True if the checksums match. False otherwise. - """ - sha256 = hashlib.sha256() - sha256.update(file.read_bytes()) - return sha256.hexdigest() == expected_checksum - - -BIN_ARCH_MAP: dict[Arch, str] = {Arch.ARM64: "arm64", Arch.X64: "amd64"} - - -def _install_external_packages(arch: Arch) -> None: +def _install_external_packages() -> None: """Install packages outside of apt. - Installs yarn, yq. - - Args: - arch: The architecture to download binaries for. #TODO check bin arch + Installs yarn. Raises: ExternalPackageInstallError: If there was an error installing external package. """ try: + # 2024/04/26 There's a potential security risk here, npm is subject to toolchain attacks. subprocess.run( ["/usr/bin/npm", "install", "--global", "yarn"], check=True, timeout=60 * 5 ) # nosec: B603 subprocess.run( ["/usr/bin/npm", "cache", "clean", "--force"], check=True, timeout=60 ) # nosec: B603 - bin_arch = BIN_ARCH_MAP[arch] - yq_path_str = f"yq_linux_{bin_arch}" - # The URLs are trusted - urllib.request.urlretrieve( - YQ_DOWNLOAD_URL_TMPL.format(BIN_ARCH=bin_arch), yq_path_str - ) # nosec: B310 - urllib.request.urlretrieve(YQ_BINARY_CHECKSUM_URL, "checksums") # nosec: B310 - urllib.request.urlretrieve( - YQ_CHECKSUM_HASHES_ORDER_URL, "checksums_hashes_order" - ) # nosec: B310 - urllib.request.urlretrieve( - YQ_EXTRACT_CHECKSUM_SCRIPT_URL, "extract-checksum.sh" - ) # nosec: B310 - # The output is - checksum = subprocess.check_output( # nosec: B603 - ["/usr/bin/bash", "extract-checksum.sh", "SHA-256", yq_path_str], - encoding="utf-8", - timeout=60, - ).split()[1] - yq_path = Path(yq_path_str) - if not _validate_checksum(yq_path, checksum): - raise ExternalPackageInstallError("Invalid checksum") - yq_path.chmod(755) - yq_path.rename("/usr/bin/yq") - except (subprocess.SubprocessError, urllib.error.ContentTooShortError) as exc: + except subprocess.SubprocessError as exc: raise ExternalPackageInstallError from exc -class ImageCompressError(Exception): - """Represents an error while compressing cloud-img.""" - - @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) -def _compress_image(image: Path) -> Path: +def _compress_image(image: Path) -> None: """Compress the cloud image. Args: @@ -465,33 +461,17 @@ def _compress_image(image: Path) -> Path: Raises: ImageCompressError: If there was something wrong compressing the image. - - Returns: - The compressed image path. """ try: subprocess.run( # nosec: B603 - ["/usr/bin/virt-sparsify", "--compress", str(image), "compressed.img"], + ["/usr/bin/virt-sparsify", "--compress", str(image), str(IMAGE_OUTPUT_PATH)], check=True, timeout=60 * 10, ) - return Path("compressed.img") except subprocess.CalledProcessError as exc: raise ImageCompressError from exc -IMAGE_DEFAULT_APT_PACKAGES = [ - "docker.io", - "npm", - "python3-pip", - "shellcheck", - "jq", - "wget", - "unzip", - "gh", -] - - @dataclasses.dataclass class BuildImageConfig: """Configuration for building the image. @@ -505,11 +485,7 @@ class BuildImageConfig: base_image: BaseImage -class BuildImageError(Exception): - """Represents an error while building the image.""" - - -def build_image(config: BuildImageConfig) -> Path: +def build_image(config: BuildImageConfig) -> None: """Build and save the image locally. Args: @@ -517,55 +493,61 @@ def build_image(config: BuildImageConfig) -> Path: Raises: BuildImageError: If there was an error building the image. - - Returns: - The saved image path. """ logger.info("Clean build state.") - _clean_build_state() - try: - logger.info("Downloading cloud image.") - cloud_image_path = _download_cloud_image(arch=config.arch, base_image=config.base_image) - logger.info("Resizing cloud image.") - _resize_cloud_img(cloud_image_path=cloud_image_path) - logger.info("Mounting network block device.") - _mount_image_to_network_block_device(cloud_image_path=cloud_image_path) - logger.info("Replacing resolv.conf.") - _replace_mounted_resolv_conf() - logger.info("Resizing partitions.") - _resize_mount_partitions() - except (CloudImageDownloadError, ImageMountError, ResizePartitionError) as exc: - raise BuildImageError from exc - - try: - logger.info("Setting up chroot environment.") - with ChrootContextManager(IMAGE_MOUNT_DIR): - # operator_libs_linux apt package uses dpkg -l and that does not work well with chroot - # env, hence use subprocess run. - subprocess.run( - ["/usr/bin/apt-get", "update", "-y"], check=True, timeout=60 * 5 - ) # nosec: B603 - subprocess.run( # nosec: B603 - ["/usr/bin/apt-get", "install", "-y", *IMAGE_DEFAULT_APT_PACKAGES], - check=True, - timeout=60 * 10, + with redirect_stdout(sys.stderr): + try: + _clean_build_state() + logger.info("Downloading cloud image.") + cloud_image_path = _download_cloud_image( + arch=config.arch, base_image=config.base_image ) - _create_python_symlinks() - _disable_unattended_upgrades() - _configure_system_users() - _install_external_packages(arch=config.arch) - except ( - ChrootBaseError, - subprocess.CalledProcessError, - UnattendedUpgradeDisableError, - SystemUserConfigurationError, - ExternalPackageInstallError, - ) as exc: - raise BuildImageError from exc - - try: - _clean_build_state() - logger.info("Compressing image") - return _compress_image(cloud_image_path) - except ImageCompressError as exc: - raise BuildImageError from exc + logger.info("Resizing cloud image.") + _resize_cloud_img(cloud_image_path=cloud_image_path) + logger.info("Mounting network block device.") + _mount_image_to_network_block_device(cloud_image_path=cloud_image_path) + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() + logger.info("Resizing partitions.") + _resize_mount_partitions() + logger.info("Building YQ from source.") + _install_yq() + except ImageBuilderBaseError as exc: + raise BuildImageError from exc + + try: + logger.info("Setting up chroot environment.") + with ChrootContextManager(IMAGE_MOUNT_DIR): + # operator_libs_linux apt package uses dpkg -l and that does not work well with + # chroot env, hence use subprocess run. + subprocess.run( + ["/usr/bin/apt-get", "update", "-y"], + check=True, + timeout=60 * 10, + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) # nosec: B603 + subprocess.run( # nosec: B603 + [ + "/usr/bin/apt-get", + "install", + "-y", + "--no-install-recommends", + *IMAGE_DEFAULT_APT_PACKAGES, + ], + check=True, + timeout=60 * 20, + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + _create_python_symlinks() + _disable_unattended_upgrades() + _configure_system_users() + _install_external_packages() + except (ImageBuilderBaseError, ChrootBaseError) as exc: + raise BuildImageError from exc + + try: + _clean_build_state() + logger.info("Compressing image") + _compress_image(cloud_image_path) + except ImageBuilderBaseError as exc: + raise BuildImageError from exc diff --git a/src/github-runner-image-builder/chroot.py b/src/github_runner_image_builder/chroot.py similarity index 98% rename from src/github-runner-image-builder/chroot.py rename to src/github_runner_image_builder/chroot.py index 7a6beee..1bd4658 100644 --- a/src/github-runner-image-builder/chroot.py +++ b/src/github_runner_image_builder/chroot.py @@ -59,6 +59,7 @@ def __enter__(self) -> None: ) except subprocess.CalledProcessError as exc: raise MountError from exc + os.chroot(self.chroot_path) os.chdir("/") @@ -90,7 +91,8 @@ def __exit__(self, *_args: Any, **_kwargs: Any) -> None: try: subprocess.run( # nosec: B603 - ["/usr/bin/umount", "-l", str(self.chroot_path / CHROOT_DEVICE_DIR)], check=True + ["/usr/bin/umount", "-l", str(self.chroot_path / CHROOT_DEVICE_DIR)], + check=True, ) except subprocess.CalledProcessError as exc: raise MountError from exc diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py new file mode 100644 index 0000000..c86a0c3 --- /dev/null +++ b/src/github_runner_image_builder/cli.py @@ -0,0 +1,199 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Main entrypoint for github-runner-image-builder cli application.""" +import argparse +import itertools + +# Subprocess module is used to execute trusted commands +import subprocess # nosec: B404 +import sys +from pathlib import Path +from typing import cast + +from github_runner_image_builder import builder +from github_runner_image_builder.builder import BuildImageConfig +from github_runner_image_builder.config import ( + IMAGE_OUTPUT_PATH, + LTS_IMAGE_VERSION_TAG_MAP, + ActionsNamespace, + BaseImage, + get_supported_arch, +) +from github_runner_image_builder.upload import OpenstackManager, UploadImageConfig + + +def _existing_path(value: str) -> Path: + """Check the path exists. + + Args: + value: The path string. + + Raises: + ValueError: If the path does not exist. + + Returns: + Path that exists. + """ + path = Path(value) + if not path.exists(): + raise ValueError(f"Given path {value} not found.") + return path + + +def _install() -> None: + """Install builder.""" + builder.setup_builder() + + +def _get(cloud_name: str, image_name: str) -> None: + """Get latest built image from OpenStack. + + Args: + cloud_name: The Openstack cloud to upload the image to. + image_name: The image name to upload as. + """ + with OpenstackManager(cloud_name=cloud_name) as manager: + sys.stdout.write(manager.get_latest_image_id(image_name=image_name)) + + +def _build_and_upload( + base: str, + callback_script_path: Path, + cloud_name: str, + image_name: str, + num_revisions: int, +) -> None: + """Build and upload image. + + Args: + base: Ubuntu image base. + callback_script_path: Path to bash script to call after image upload. + cloud_name: The Openstack cloud to upload the image to. + image_name: The image name to upload as. + num_revisions: Number of image revisions to keep before deletion. + """ + arch = get_supported_arch() + base_image = BaseImage.from_str(base) + build_config = BuildImageConfig(arch=arch, base_image=base_image) + builder.build_image(config=build_config) + with OpenstackManager(cloud_name=cloud_name) as manager: + image_id = manager.upload_image( + config=UploadImageConfig( + arch=arch, + base=base_image, + image_name=image_name, + num_revisions=num_revisions, + src_path=IMAGE_OUTPUT_PATH, + ) + ) + # The callback script is a user trusted script. + subprocess.check_call(["/bin/bash", str(callback_script_path), image_id]) # nosec: B603 + + +def main(args: list[str] | None = None) -> None: + """Run entrypoint for Github runner image builder CLI. + + Args: + args: Command line arguments. + """ + # The following line is used for unit testing. + if args is None: # pragma: nocover + args = sys.argv[1:] + + parser = argparse.ArgumentParser( + prog="Github runner image builder CLI", + description="Builds github runner image and uploads it to openstack.", + ) + subparsers = parser.add_subparsers( + title="actions", + description="command modes for Github runner image builder CLI.", + dest="action", + required=True, + ) + subparsers.add_parser("install") + get_parser = subparsers.add_parser("get") + get_parser.add_argument( + "-c", + "--cloud-name", + dest="cloud_name", + required=True, + help=( + "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " + "of the following order: current directory, ~/.config/openstack, /etc/openstack." + ), + ) + get_parser.add_argument( + "-o", + "--output-image-name", + dest="image_name", + required=True, + help="The image name uploaded to Openstack.", + ) + build_parser = subparsers.add_parser("build") + build_parser.add_argument( + "-i", + "--image-base", + dest="base", + required=False, + choices=tuple( + itertools.chain.from_iterable( + (tag, name) for (tag, name) in LTS_IMAGE_VERSION_TAG_MAP.items() + ) + ), + default="jammy", + ) + build_parser.add_argument( + "-c", + "--cloud-name", + dest="cloud_name", + required=True, + help=( + "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " + "of the following order: current directory, ~/.config/openstack, /etc/openstack." + ), + ) + build_parser.add_argument( + "-n", + "--num-revisions", + dest="num_revisions", + required=False, + type=int, + default=5, + help="The maximum number of images to keep before deletion.", + ) + build_parser.add_argument( + "-p", + "--callback-script-path", + dest="callback_script_path", + required=True, + type=_existing_path, + help=( + "The callback script to trigger after image is built. The callback script is called" + "with the first argument as the image ID." + ), + ) + build_parser.add_argument( + "-o", + "--output-image-name", + dest="image_name", + required=True, + help="The image name to upload to Openstack.", + ) + parsed = cast(ActionsNamespace, parser.parse_args(args)) + + if parsed.action == "install": + _install() + return + + if parsed.action == "get": + _get(cloud_name=parsed.cloud_name, image_name=parsed.image_name) + return + + _build_and_upload( + base=parsed.base, + callback_script_path=parsed.callback_script_path, + cloud_name=parsed.cloud_name, + image_name=parsed.image_name, + num_revisions=parsed.num_revisions, + ) diff --git a/src/github_runner_image_builder/config.py b/src/github_runner_image_builder/config.py new file mode 100644 index 0000000..70279c5 --- /dev/null +++ b/src/github_runner_image_builder/config.py @@ -0,0 +1,125 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module containing configurations.""" + +import argparse +import logging +import platform +from enum import Enum +from pathlib import Path +from typing import Literal + +logger = logging.getLogger(__name__) + + +# This is a class used for type hinting argparse. +class ActionsNamespace(argparse.Namespace): # pylint: disable=too-few-public-methods + """Action positional argument namespace. + + Attributes: + action: CLI action positional argument. + base: The base image to build. + callback_script_path: The callback script path to run after image build. + cloud_name: The Openstack cloud to interact with. The CLI assumes clouds.yaml is written + to the default path, i.e. current directory or ~/.config/openstack or /etc/openstack. + image_name: The image name to upload as. + num_revisions: The maximum number of images to keep before deletion. + """ + + action: Literal["install", "build", "get"] + base: Literal["22.04", "jammy", "24.04", "noble"] + callback_script_path: Path + cloud_name: str + image_name: str + num_revisions: int + + +class Arch(str, Enum): + """Supported system architectures. + + Attributes: + ARM64: Represents an ARM64 system architecture. + X64: Represents an X64/AMD64 system architecture. + """ + + ARM64 = "arm64" + X64 = "x64" + + +class UnsupportedArchitectureError(Exception): + """Raised when given machine architecture is unsupported. + + Attributes: + arch: The current machine architecture. + """ + + def __str__(self) -> str: + """Represent the error in string format. + + Returns: + The error in string format. + """ + return f"UnsupportedArchitectureError: {self.arch}" + + def __init__(self, arch: str) -> None: + """Initialize a new instance of the UnsupportedArchitectureError exception. + + Args: + arch: The current machine architecture. + """ + self.arch = arch + + +ARCHITECTURES_ARM64 = {"aarch64", "arm64"} +ARCHITECTURES_X86 = {"x86_64"} + + +def get_supported_arch() -> Arch: + """Get current machine architecture. + + Raises: + UnsupportedArchitectureError: if the current architecture is unsupported. + + Returns: + Arch: Current machine architecture. + """ + arch = platform.machine() + match arch: + case arch if arch in ARCHITECTURES_ARM64: + return Arch.ARM64 + case arch if arch in ARCHITECTURES_X86: + return Arch.X64 + case _: + raise UnsupportedArchitectureError(arch=arch) + + +class BaseImage(str, Enum): + """The ubuntu OS base image to build and deploy runners on. + + Attributes: + JAMMY: The jammy ubuntu LTS image. + NOBLE: The noble ubuntu LTS image. + """ + + JAMMY = "jammy" + NOBLE = "noble" + + @classmethod + def from_str(cls, tag_or_name: str) -> "BaseImage": + """Retrieve the base image tag from input. + + Args: + tag_or_name: The base image string option. + + Returns: + The base image configuration of the app. + """ + if tag_or_name in LTS_IMAGE_VERSION_TAG_MAP: + return cls(LTS_IMAGE_VERSION_TAG_MAP[tag_or_name]) + return cls(tag_or_name) + + +LTS_IMAGE_VERSION_TAG_MAP = {"22.04": BaseImage.JAMMY.value, "24.04": BaseImage.NOBLE.value} + +IMAGE_OUTPUT_PATH = Path("compressed.img") diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py new file mode 100644 index 0000000..8050ff7 --- /dev/null +++ b/src/github_runner_image_builder/errors.py @@ -0,0 +1,89 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module containing error definitions.""" + + +class ImageBuilderBaseError(Exception): + """Represents an error with any builder related executions.""" + + +# nosec: B603: All subprocess runs are run with trusted executables. +class DependencyInstallError(ImageBuilderBaseError): + """Represents an error while installing required dependencies.""" + + +class NetworkBlockDeviceError(ImageBuilderBaseError): + """Represents an error while enabling network block device.""" + + +class BuilderSetupError(ImageBuilderBaseError): + """Represents an error while setting up host machine as builder.""" + + +class UnsupportedArchitectureError(ImageBuilderBaseError): + """Raised when given machine architecture is unsupported.""" + + +class CleanBuildStateError(ImageBuilderBaseError): + """Represents an error cleaning up build state.""" + + +class CloudImageDownloadError(ImageBuilderBaseError): + """Represents an error downloading cloud image.""" + + +class ImageResizeError(ImageBuilderBaseError): + """Represents an error while resizing the image.""" + + +class ImageMountError(ImageBuilderBaseError): + """Represents an error while mounting the image to network block device.""" + + +class ResizePartitionError(ImageBuilderBaseError): + """Represents an error while resizing network block device partitions.""" + + +class UnattendedUpgradeDisableError(ImageBuilderBaseError): + """Represents an error while disabling unattended-upgrade related services.""" + + +class SystemUserConfigurationError(ImageBuilderBaseError): + """Represents an error while adding user to chroot env.""" + + +class YQBuildError(ImageBuilderBaseError): + """Represents an error while building yq binary from source.""" + + +class ExternalPackageInstallError(ImageBuilderBaseError): + """Represents an error installilng external packages.""" + + +class ImageCompressError(ImageBuilderBaseError): + """Represents an error while compressing cloud-img.""" + + +class BuildImageError(ImageBuilderBaseError): + """Represents an error while building the image.""" + + +class OpenstackBaseError(Exception): + """Represents an error while interacting with Openstack.""" + + +class UnauthorizedError(OpenstackBaseError): + """Represents an unauthorized connection to Openstack.""" + + +class UploadImageError(OpenstackBaseError): + """Represents an error when uploading image to Openstack.""" + + +class GetImageError(OpenstackBaseError): + """Represents an error when fetching images from Openstack.""" + + +class OpenstackConnectionError(OpenstackBaseError): + """Represents an error while communicating with Openstack.""" diff --git a/src/github_runner_image_builder/upload.py b/src/github_runner_image_builder/upload.py new file mode 100644 index 0000000..bdd2b18 --- /dev/null +++ b/src/github_runner_image_builder/upload.py @@ -0,0 +1,159 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for uploading images to shareable storage.""" + +import dataclasses +import logging +from pathlib import Path +from typing import Any, cast + +import openstack +import openstack.connection +import openstack.exceptions +from openstack.image.v2.image import Image + +from github_runner_image_builder.config import Arch, BaseImage +from github_runner_image_builder.errors import ( + GetImageError, + OpenstackConnectionError, + UnauthorizedError, + UploadImageError, +) + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class UploadImageConfig: + """Configuration values for creating image. + + Attributes: + arch: The architecture the image was built for. + base: The ubuntu OS base the image was created with. + image_name: The image name to upload as. + num_revisions: The number of revisions to keep for an image. + src_path: The path to image to upload. + """ + + arch: Arch + base: BaseImage + image_name: str + num_revisions: int + src_path: Path + + +class OpenstackManager: + """Class to manage interactions with Openstack.""" + + def __init__(self, cloud_name: str): + """Initialize the openstack manager class. + + Args: + cloud_name: The Openstack cloud to use. + + Raises: + UnauthorizedError: If an invalid openstack credentials was given. + """ + try: + with openstack.connect(cloud=cloud_name) as conn: + conn.authorize() + # pylint thinks this isn't an exception, but does inherit from Exception class. + except openstack.exceptions.HttpException as exc: # pylint: disable=bad-exception-cause + raise UnauthorizedError("Unauthorized credentials.") from exc + + self.conn = openstack.connect(cloud_name) + + def __enter__(self) -> "OpenstackManager": + """Dunder method placeholder for context management. + + Returns: + Self with established connection. + """ + return self + + def __exit__(self, *_args: Any, **_kwargs: Any) -> None: + """Dunder method to close initialized connection to openstack.""" + self.conn.close() + + def _get_images_by_latest(self, image_name: str) -> list[Image]: + """Fetch the images sorted by latest. + + Args: + image_name: The image name to search for. + + Raises: + OpenstackConnectionError: if there was an error fetching the images. + + Returns: + The images sorted in latest first order. + """ + try: + images = cast(list[Image], self.conn.search_images(image_name)) + except openstack.exceptions.OpenStackCloudException as exc: + raise OpenstackConnectionError from exc + + return sorted(images, key=lambda image: image.created_at, reverse=True) + + def _prune_old_images(self, image_name: str, num_revisions: int) -> None: + """Remove old images outside of number of revisions to keep. + + Args: + image_name: The image name to search for. + num_revisions: The number of revisions to keep. + """ + images = self._get_images_by_latest(image_name=image_name) + images_to_prune = images[num_revisions:] + for image in images_to_prune: + try: + if not self.conn.delete_image(image.id, wait=True): + logger.error("Failed to delete old image, %s", image.id) + except openstack.exceptions.OpenStackCloudException as exc: + logger.error("Failed to prune old image, %s", exc) + continue + + def upload_image(self, config: UploadImageConfig) -> str: + """Upload image to openstack glance. + + Args: + config: Configuration values for creating image. + + Raises: + UploadImageError: If there was an error uploading the image to Openstack Glance. + + Returns: + The created image ID. + """ + try: + self._prune_old_images( + image_name=config.image_name, num_revisions=config.num_revisions - 1 + ) + image: Image = self.conn.create_image( + name=config.image_name, + filename=str(config.src_path), + allow_duplicates=True, + wait=True, + ) + return image.id + except openstack.exceptions.OpenStackCloudException as exc: + raise UploadImageError from exc + + def get_latest_image_id(self, image_name: str) -> str | None: + """Fetch the latest image id. + + Args: + image_name: The image name to search for. + + Raises: + GetImageError: If there was an error fetching image from Openstack. + + Returns: + The image ID if exists, None otherwise. + """ + try: + images = self._get_images_by_latest(image_name=image_name) + except OpenstackConnectionError as exc: + raise GetImageError from exc + if not images: + return None + return images[0].id diff --git a/src/github-runner-image-builder/utils.py b/src/github_runner_image_builder/utils.py similarity index 98% rename from src/github-runner-image-builder/utils.py rename to src/github_runner_image_builder/utils.py index daae1f4..47d8980 100644 --- a/src/github-runner-image-builder/utils.py +++ b/src/github_runner_image_builder/utils.py @@ -1,7 +1,7 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -"""Utilities used by the charm.""" +"""Utilities used by the app.""" import functools import logging diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b06afc8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,57 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for github runner image builder app.""" + + +from pytest import Parser + + +def pytest_addoption(parser: Parser): + """Add options to pytest parser. + + Args: + parser: The pytest argument parser. + """ + parser.addoption("--image", action="store", help="The Ubuntu LTS base image to build.") + parser.addoption( + "--openstack-clouds-yaml", + action="store", + help="The OpenStack clouds yaml contents the charm uses to connect to Openstack.", + ) + # Private endpoint options + parser.addoption( + "--openstack-auth-url", + action="store", + help="The URL to Openstack authentication service, i.e. keystone.", + ) + parser.addoption( + "--openstack-password", + action="store", + help="The password to authenticate to Openstack service.", + ) + parser.addoption( + "--openstack-project-domain-name", + action="store", + help="The Openstack project domain name to use.", + ) + parser.addoption( + "--openstack-project-name", + action="store", + help="The Openstack project name to use.", + ) + parser.addoption( + "--openstack-user-domain-name", + action="store", + help="The Openstack user domain name to use.", + ) + parser.addoption( + "--openstack-user-name", + action="store", + help="The Openstack user to authenticate as.", + ) + parser.addoption( + "--openstack-region-name", + action="store", + help="The Openstack region to authenticate to.", + ) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..887614c --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests module.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..82f4c6f --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,155 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for github runner image builder integration tests.""" +import string +from pathlib import Path +from typing import Optional + +import openstack +import pytest +import yaml +from openstack.connection import Connection + +from github_runner_image_builder.cli import main + + +@pytest.fixture(scope="module", name="image") +def image_fixture(pytestconfig: pytest.Config) -> str: + """The ubuntu image base to build from.""" + image = pytestconfig.getoption("--image") + assert image, "Please specify the --image command line option" + return image + + +@pytest.fixture(scope="module", name="openstack_clouds_yaml") +def openstack_clouds_yaml_fixture(pytestconfig: pytest.Config) -> str: + """Configured clouds-yaml setting.""" + clouds_yaml = pytestconfig.getoption("--openstack-clouds-yaml") + return clouds_yaml + + +@pytest.fixture(scope="module", name="private_endpoint_clouds_yaml") +def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> Optional[str]: + """The openstack private endpoint clouds yaml.""" + auth_url = pytestconfig.getoption("--openstack-auth-url") + password = pytestconfig.getoption("--openstack-password") + project_domain_name = pytestconfig.getoption("--openstack-project-domain-name") + project_name = pytestconfig.getoption("--openstack-project-name") + user_domain_name = pytestconfig.getoption("--openstack-user-domain-name") + user_name = pytestconfig.getoption("--openstack-user-name") + region_name = pytestconfig.getoption("--openstack-region-name") + if any( + not val + for val in ( + auth_url, + password, + project_domain_name, + project_name, + user_domain_name, + user_name, + region_name, + ) + ): + return None + return string.Template( + Path("tests/integration/data/clouds.yaml.tmpl").read_text(encoding="utf-8") + ).substitute( + { + "auth_url": auth_url, + "password": password, + "project_domain_name": project_domain_name, + "project_name": project_name, + "user_domain_name": user_domain_name, + "username": user_name, + "region_name": region_name, + } + ) + + +@pytest.fixture(scope="module", name="clouds_yaml_contents") +def clouds_yaml_contents_fixture( + openstack_clouds_yaml: Optional[str], private_endpoint_clouds_yaml: Optional[str] +): + """The Openstack clouds yaml or private endpoint cloud yaml contents.""" + clouds_yaml_contents = openstack_clouds_yaml or private_endpoint_clouds_yaml + assert clouds_yaml_contents, ( + "Please specify --openstack-clouds-yaml or all of private endpoint arguments " + "(--openstack-auth-url, --openstack-password, --openstack-project-domain-name, " + "--openstack-project-name, --openstack-user-domain-name, --openstack-user-name, " + "--openstack-region-name)" + ) + return clouds_yaml_contents + + +@pytest.fixture(scope="module", name="cloud_name") +def cloud_name_fixture(clouds_yaml_contents: str) -> str: + """The cloud to use from cloud config.""" + clouds_yaml = yaml.safe_load(clouds_yaml_contents) + clouds_yaml_path = Path("clouds.yaml") + clouds_yaml_path.write_text(data=clouds_yaml_contents, encoding="utf-8") + first_cloud = next(iter(clouds_yaml["clouds"].keys())) + return first_cloud + + +@pytest.fixture(scope="module", name="openstack_connection") +def openstack_connection_fixture(cloud_name: str) -> Connection: + """The openstack connection instance.""" + return openstack.connect(cloud_name) + + +@pytest.fixture(scope="module", name="callback_result_path") +def callback_result_path_fixture() -> Path: + """The file created when the callback script is run.""" + return Path("callback_complete") + + +@pytest.fixture(scope="module", name="callback_script") +def callback_script_fixture(callback_result_path: Path) -> Path: + """The callback script to use with the image builder.""" + callback_script = Path("callback") + callback_script.write_text( + f"""#!/bin/bash +IMAGE_ID=$1 +echo $IMAGE_ID | tee {callback_result_path} +""", + encoding="utf-8", + ) + return callback_script + + +@pytest.fixture(scope="module", name="openstack_image_name") +def openstack_image_name_fixture() -> str: + """The image name to upload to openstack.""" + return "image-builder-test-image" + + +@pytest.fixture(scope="module", name="cli_run") +def cli_run_fixture( + image: str, + cloud_name: str, + callback_script: Path, + openstack_connection: Connection, + openstack_image_name: str, +): + """A CLI run.""" + main(["install"]) + main( + [ + "build", + "-i", + image, + "-c", + cloud_name, + "-n", + "2", + "-p", + str(callback_script), + "-o", + openstack_image_name, + ] + ) + + yield + + openstack_connection.delete_image(openstack_image_name) diff --git a/tests/integration/data/clouds.yaml.tmpl b/tests/integration/data/clouds.yaml.tmpl new file mode 100644 index 0000000..3b6cc91 --- /dev/null +++ b/tests/integration/data/clouds.yaml.tmpl @@ -0,0 +1,10 @@ +clouds: + testcloud: + auth: + auth_url: ${auth_url} + password: ${password} + project_domain_name: ${project_domain_name} + project_name: ${project_name} + user_domain_name: ${user_domain_name} + username: ${username} + region_name: ${region_name} diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..6a26c9f --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,204 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Helper utilities for integration tests.""" + +import collections +import inspect +import platform +import tarfile +import time +from functools import partial +from pathlib import Path +from string import Template +from typing import Awaitable, Callable, ParamSpec, TypeVar, cast + +from pylxd import Client +from pylxd.models.image import Image +from pylxd.models.instance import Instance, InstanceState +from requests_toolbelt import MultipartEncoder + +P = ParamSpec("P") +R = TypeVar("R") +S = Callable[P, R] | Callable[P, Awaitable[R]] + + +async def wait_for( + func: S, + timeout: int | float = 300, + check_interval: int = 10, +) -> R: + """Wait for function execution to become truthy. + + Args: + func: A callback function to wait to return a truthy value. + timeout: Time in seconds to wait for function result to become truthy. + check_interval: Time in seconds to wait between ready checks. + + Raises: + TimeoutError: if the callback function did not return a truthy value within timeout. + + Returns: + The result of the function if any. + """ + deadline = time.time() + timeout + is_awaitable = inspect.iscoroutinefunction(func) + while time.time() < deadline: + if is_awaitable: + if result := await cast(Awaitable, func()): + return result + else: + if result := func(): + return cast(R, result) + time.sleep(check_interval) + + # final check before raising TimeoutError. + if is_awaitable: + if result := await cast(Awaitable, func()): + return result + else: + if result := func(): + return cast(R, result) + raise TimeoutError() + + +IMAGE_TO_TAG = {"jammy": "22.04", "noble": "24.04"} + + +def _create_metadata_tar_gz(image: str, tmp_path: Path) -> Path: + """Create metadata.tar.gz contents. + + Args: + image: The ubuntu LTS image name. + tmp_path: Temporary dir. + """ + # Create metadata.yaml + template = Template( + Path("tests/integration/testdata/metadata.yaml.tmpl").read_text(encoding="utf-8") + ) + metadata_contents = template.substitute( + {"arch": platform.machine(), "tag": IMAGE_TO_TAG[image], "image": image} + ) + meta_path = tmp_path / "metadata.yaml" + meta_path.write_text(metadata_contents, encoding="utf-8") + + # Pack templates/ and metada.yaml + templates_path = Path("tests/integration/testdata/templates") + metadata_tar = tmp_path / Path("metadata.tar.gz") + + with tarfile.open(metadata_tar, "w:gz") as tar: + tar.add(meta_path, arcname=meta_path.name) + tar.add(templates_path, arcname=templates_path.name) + + return metadata_tar + + +# This is a workaround until https://github.com/canonical/pylxd/pull/577 gets merged. +def _post_vm_img( + client: Client, + image_data: bytes, + metadata: bytes | None = None, + public: bool = False, +) -> Image: + """Create an LXD VM image. + + Args: + client: The LXD client. + image_data: Image qcow2 (.img) file contents in bytes. + metadata: The metadata.tar.gz contents in bytes. + public: Whether the image should be publicly available. + """ + headers = {} + if public: + headers["X-LXD-Public"] = "1" + + if metadata is not None: + # Image uploaded as chunked/stream (metadata, rootfs) + # multipart message. + # Order of parts is important metadata should be passed first + files = collections.OrderedDict( + { + "metadata": ("metadata", metadata, "application/octet-stream"), + # rootfs is container, rootfs.img is VM + "rootfs.img": ("rootfs.img", image_data, "application/octet-stream"), + } + ) + data = MultipartEncoder(files) + headers.update({"Content-Type": data.content_type}) + else: + data = image_data + + response = client.api.images.post(data=data, headers=headers) + operation = client.operations.wait_for_operation(response.json()["operation"]) + return Image(client, fingerprint=operation.metadata["fingerprint"]) + + +def create_lxd_vm_image(lxd_client: Client, img_path: Path, image: str, tmp_path: Path) -> Image: + """Create LXD VM image. + + 1. Creates the metadata.tar.gz file with the corresponding Ubuntu OS image and a pre-defined + templates directory. See testdata/templates. + 2. Uploads the created VM image to LXD - metadata and image of qcow2 format is required. + 3. Tags the uploaded image with an alias for test use. + + Args: + lxd_client: PyLXD client. + img_path: qcow2 (.img) file path to upload. + tmp_path: Temporary dir. + image: The Ubuntu image name. + + Returns: + The created LXD image. + """ + metadata_tar = _create_metadata_tar_gz(image=image, tmp_path=tmp_path) + lxd_image = _post_vm_img( + lxd_client, img_path.read_bytes(), metadata_tar.read_bytes(), public=True + ) + lxd_image.add_alias(image, f"Ubuntu {image} {IMAGE_TO_TAG[image]} image.") + return lxd_image + + +def _instance_running(instance: Instance) -> bool: + """Check if the instance is running. + + Args: + instance: The lxd instance. + + Returns: + Whether the instance is running. + """ + state: InstanceState = instance.state() + if state.status != "Running": + return False + try: + result = instance.execute( + ["sudo", "--user", "ubuntu", "sudo", "systemctl", "is-active", "snapd.seeded.service"] + ) + except BrokenPipeError: + return False + return result.exit_code == 0 + + +async def create_lxd_instance(lxd_client: Client, image: str) -> Instance: + """Create and wait for LXD instance to become active. + + Args: + lxd_client: PyLXD client. + image: The Ubuntu image name. + + Returns: + The created and running LXD instance. + """ + instance_config = { + "name": f"test-{image}", + "source": {"type": "image", "alias": image}, + "type": "virtual-machine", + "config": {"limits.cpu": "3", "limits.memory": "8192MiB"}, + } + instance: Instance = lxd_client.instances.create( # pylint: disable=no-member + instance_config, wait=True + ) + instance.start(timeout=10 * 60, wait=True) + await wait_for(partial(_instance_running, instance)) + + return instance diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 0000000..0c7f3b3 --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1 @@ +pylxd diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py new file mode 100644 index 0000000..7096cd6 --- /dev/null +++ b/tests/integration/test_image.py @@ -0,0 +1,137 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Image test module.""" + +import logging +from pathlib import Path +from typing import NamedTuple + +import pytest +from openstack.connection import Connection +from pylxd import Client + +from github_runner_image_builder.cli import main +from github_runner_image_builder.config import IMAGE_OUTPUT_PATH +from tests.integration.helpers import create_lxd_instance, create_lxd_vm_image + +logger = logging.getLogger(__name__) + + +class Commands(NamedTuple): + """Test commands to execute. + + Attributes: + name: The test name. + command: The command to execute. + """ + + name: str + command: str + + +# This is matched with E2E test run of github-runner-operator charm. +TEST_RUNNER_COMMANDS = ( + Commands(name="simple hello world", command="echo hello world"), + Commands(name="file permission to /usr/local/bin", command="ls -ld /usr/local/bin"), + Commands( + name="file permission to /usr/local/bin (create)", command="touch /usr/local/bin/test_file" + ), + Commands(name="install microk8s", command="sudo snap install microk8s --classic"), + Commands(name="wait for microk8s", command="microk8s status --wait-ready"), + Commands( + name="deploy nginx in microk8s", + command="microk8s kubectl create deployment nginx --image=nginx", + ), + Commands( + name="wait for nginx", + command="microk8s kubectl rollout status deployment/nginx --timeout=20m", + ), + Commands(name="update apt in docker", command="docker run python:3.10-slim apt-get update"), + Commands(name="docker version", command="docker version"), + Commands(name="check python3 alias", command="python --version"), + Commands(name="pip version", command="python3 -m pip --version"), + Commands(name="npm version", command="npm --version"), + Commands(name="shellcheck version", command="shellcheck --version"), + Commands(name="jq version", command="jq --version"), + Commands(name="yq version", command="yq --version"), + Commands(name="apt update", command="sudo apt-get update -y"), + Commands(name="install pipx", command="sudo apt-get install -y pipx"), + Commands(name="pipx add path", command="pipx ensurepath"), + Commands(name="install check-jsonschema", command="pipx install check-jsonschema"), + Commands(name="check jsonschema", command="check-jsonschema --version"), + Commands(name="unzip version", command="unzip -v"), + Commands(name="gh version", command="gh --version"), + Commands( + name="test sctp support", command="sudo apt-get install lksctp-tools -yq && checksctp" + ), +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cli_run") +async def test_image(image: str, tmp_path: Path): + """ + arrange: given a built output from the CLI. + act: when the image is booted and commands are executed. + assert: commands do not error. + """ + lxd = Client() + logger.info("Creating LXD VM Image.") + create_lxd_vm_image(lxd_client=lxd, img_path=IMAGE_OUTPUT_PATH, image=image, tmp_path=tmp_path) + logger.info("Launching LXD instance.") + instance = await create_lxd_instance(lxd_client=lxd, image=image) + + for testcmd in TEST_RUNNER_COMMANDS: + logger.info("Running command: %s", testcmd.command) + # run command as ubuntu user. Passing in user argument would not be equivalent to a login + # shell which is missing critical environment variables such as $USER and the user groups + # are not properly loaded. + result = instance.execute( + ["su", "--shell", "/bin/bash", "--login", "ubuntu", "-c", testcmd.command] + ) + logger.info("Command output: %s %s %s", result.exit_code, result.stdout, result.stderr) + assert result.exit_code == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cli_run") +async def test_openstack_upload(openstack_connection: Connection, openstack_image_name: str): + """ + arrange: given a built output from the CLI. + act: when openstack images are listed. + assert: the built image is uploaded in Openstack. + """ + assert len(openstack_connection.search_images(openstack_image_name)) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cli_run") +async def test_script_callback(callback_result_path: Path): + """ + arrange: given a CLI run with script that creates a file. + act: None. + assert: the file exist. + """ + assert callback_result_path.exists() + assert len(callback_result_path.read_text(encoding="utf-8")) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cli_run") +async def test_get_image( + cloud_name: str, + openstack_image_name: str, + capsys: pytest.CaptureFixture, + openstack_connection: Connection, +): + """ + arrange: a cli that already ran. + act: when get image id is run. + assert: the latest image matches the stdout output. + """ + main(["get", "-c", cloud_name, "-o", openstack_image_name]) + image_id = openstack_connection.get_image_id(openstack_image_name) + + res = capsys.readouterr() + assert res.out == image_id, f"Openstack image not matching, {res.out} {res.err}, {image_id}" diff --git a/tests/integration/testdata/metadata.yaml.tmpl b/tests/integration/testdata/metadata.yaml.tmpl new file mode 100644 index 0000000..42d3bcd --- /dev/null +++ b/tests/integration/testdata/metadata.yaml.tmpl @@ -0,0 +1,13 @@ +architecture: "$arch" +creation_date: 1713785675 +properties: + architecture: "$arch" + description: "Ubuntu $tag LTS server (20240422)" + os: "ubuntu" + release: "$image" +templates: + /etc/hostname: + when: + - create + - copy + template: hostname.tpl diff --git a/tests/integration/testdata/templates/hostname.tpl b/tests/integration/testdata/templates/hostname.tpl new file mode 100644 index 0000000..69a84f1 --- /dev/null +++ b/tests/integration/testdata/templates/hostname.tpl @@ -0,0 +1 @@ +{{ container.name }} diff --git a/src/github-runner-image-builder/__init__.py b/tests/unit/__init__.py similarity index 75% rename from src/github-runner-image-builder/__init__.py rename to tests/unit/__init__.py index c5ac4b8..3b60682 100644 --- a/src/github-runner-image-builder/__init__.py +++ b/tests/unit/__init__.py @@ -1,3 +1,4 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. +"""Unit tests module.""" diff --git a/tests/unit/factories.py b/tests/unit/factories.py new file mode 100644 index 0000000..c198acc --- /dev/null +++ b/tests/unit/factories.py @@ -0,0 +1,37 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Factories for generating test data.""" + +from typing import Generic, TypeVar +from unittest.mock import MagicMock + +import factory +from factory.faker import Faker + +T = TypeVar("T") + + +# DC060: Docstrings have been abbreviated for factories, checking for docstrings on model +# attributes can be skipped. + + +class BaseMetaFactory(Generic[T], factory.base.FactoryMetaClass): + """Used for type hints of factories.""" + + # No need for docstring because it is used for type hints + def __call__(cls, *args, **kwargs) -> T: # noqa: N805 + """Used for type hints of factories.""" # noqa: DCO020 + return super().__call__(*args, **kwargs) # noqa: DCO030 + + +class MockOpenstackImageFactory(factory.Factory): + """Mock Openstack Image.""" # noqa: DCO060 + + class Meta: # pylint: disable=too-few-public-methods + """Configuration for factory.""" # noqa: DCO060 + + model = MagicMock + + id: str # UUID + created_at = Faker("date") # Example format: 2024-04-16T04:31:12Z diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt new file mode 100644 index 0000000..c9c3f26 --- /dev/null +++ b/tests/unit/requirements.txt @@ -0,0 +1 @@ +factory-boy>=3,<4 diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py new file mode 100644 index 0000000..852eab3 --- /dev/null +++ b/tests/unit/test_builder.py @@ -0,0 +1,565 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for builder module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +import time +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from github_runner_image_builder import builder +from github_runner_image_builder.builder import ( + Arch, + BaseImage, + BuilderSetupError, + BuildImageError, + ChrootBaseError, + CleanBuildStateError, + CloudImageDownloadError, + DependencyInstallError, + ExternalPackageInstallError, + ImageCompressError, + ImageMountError, + ImageResizeError, + NetworkBlockDeviceError, + ResizePartitionError, + SupportedCloudImageArch, + SystemUserConfigurationError, + UnattendedUpgradeDisableError, + UnsupportedArchitectureError, + YQBuildError, + os, + shutil, + subprocess, +) + + +def test__install_dependencies_package_not_found(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given apt.add_package that raises PackageNotFoundError. + act: when _install_dependencies is called. + assert: DependencyInstallError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=[None, None, subprocess.CalledProcessError(1, [], "Package not found.")] + ), + ) + + with pytest.raises(DependencyInstallError) as exc: + builder._install_dependencies() + + assert "Package not found" in str(exc.getrepr()) + + +def test__enable_nbd_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given subprocess run that raises CalledProcessError. + act: when _enable_nbd is called. + assert: NetworkBlockDeviceError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=subprocess.CalledProcessError(1, [], "Module nbd not found")), + ) + + with pytest.raises(NetworkBlockDeviceError) as exc: + builder._enable_nbd() + + assert "Module nbd not found" in str(exc.getrepr()) + + +@pytest.mark.parametrize( + "patch_obj, sub_func, exception, expected_message", + [ + pytest.param( + builder, + "_install_dependencies", + DependencyInstallError("Dependency not found"), + "Dependency not found", + id="Dependency not found", + ), + pytest.param( + builder, + "_enable_nbd", + NetworkBlockDeviceError("Unable to enable nbd"), + "Unable to enable nbd", + id="Failed to enable nbd", + ), + ], +) +def test_setup_builder_fail( + patch_obj: Any, + sub_func: str, + exception: Exception, + expected_message: str, + monkeypatch: pytest.MonkeyPatch, +): + """ + arrange: given a monkeypatched sub functions of setup_builder that raises given exceptions. + act: when setup_builder is called. + assert: A BuilderSetupError is raised. + """ + mock_func = MagicMock(side_effect=exception) + monkeypatch.setattr(builder, "_install_dependencies", MagicMock) + monkeypatch.setattr(builder, "_enable_nbd", MagicMock) + monkeypatch.setattr(patch_obj, sub_func, mock_func) + + with pytest.raises(BuilderSetupError) as exc: + builder.setup_builder() + + assert expected_message in str(exc.getrepr()) + + +def test__get_supported_runner_arch_unsupported_error(): + """ + arrange: given an architecture value that isn't supported. + act: when _get_supported_runner_arch is called. + assert: UnsupportedArchitectureError is raised. + """ + arch = MagicMock() + with pytest.raises(UnsupportedArchitectureError): + builder._get_supported_runner_arch(arch) + + +@pytest.mark.parametrize( + "arch, expected", + [ + pytest.param(Arch.ARM64, "arm64", id="ARM64"), + pytest.param(Arch.X64, "amd64", id="AMD64"), + ], +) +def test__get_supported_runner_arch(arch: Arch, expected: SupportedCloudImageArch): + """ + arrange: given an architecture value that is supported. + act: when _get_supported_runner_arch is called. + assert: Expected architecture in cloud_images format is returned. + """ + assert builder._get_supported_runner_arch(arch) == expected + + +def test__clean_build_state_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a magic mocked IMAGE_MOUNT_DIR and subprocess call that raises exceptions. + act: when _clean_build_state is called. + assert: CleanBuildStateError is raised. + """ + mock_mount_dir = MagicMock() + mock_subprocess_run = MagicMock( + side_effect=subprocess.CalledProcessError(1, [], "", "qemu-nbd error") + ) + monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", mock_mount_dir) + monkeypatch.setattr(subprocess, "run", mock_subprocess_run) + + with pytest.raises(CleanBuildStateError) as exc: + builder._clean_build_state() + + assert "qemu-nbd error" in str(exc.getrepr()) + + +def test__clean_build_state(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a magic mocked IMAGE_MOUNT_DIR and qemu-nbd subprocess call. + act: when _clean_build_state is called. + assert: the mocks are called. + """ + mock_mount_dir = MagicMock() + mock_subprocess_run = MagicMock() + monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", mock_mount_dir) + monkeypatch.setattr(builder.subprocess, "run", mock_subprocess_run) + + builder._clean_build_state() + + mock_mount_dir.mkdir.assert_called_once() + mock_subprocess_run.assert_called() + + +@pytest.mark.parametrize( + "patch_obj, sub_func, exception, expected_message", + [ + pytest.param( + builder, + "_get_supported_runner_arch", + UnsupportedArchitectureError("Unsupported architecture"), + "Unsupported architecture", + id="Unsupported architecture", + ), + pytest.param( + builder.urllib.request, + "urlretrieve", + builder.urllib.error.ContentTooShortError("Network interrupted", ""), + "Network interrupted", + id="Network interrupted", + ), + ], +) +def test__download_cloud_image_fail( + patch_obj: Any, + sub_func: str, + exception: Exception, + expected_message: str, + monkeypatch: pytest.MonkeyPatch, +): + """ + arrange: given monkeypatched sub functions of _download_cloud_image that raises exceptions. + act: when _download_cloud_image is called. + assert: A CloudImageDownloadError is raised. + """ + mock_func = MagicMock(side_effect=exception) + monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock) + monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock) + monkeypatch.setattr(patch_obj, sub_func, mock_func) + + with pytest.raises(CloudImageDownloadError) as exc: + builder._download_cloud_image(arch=MagicMock(), base_image=MagicMock()) + + assert expected_message in str(exc.getrepr()) + + +def test__download_cloud_image(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given monkeypatched sub functions of _download_cloud_image. + act: when _download_cloud_image is called. + assert: the downloaded path is returned. + """ + monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock(return_value="amd64")) + monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock()) + + assert builder._download_cloud_image(arch=Arch.X64, base_image=BaseImage.JAMMY) == Path( + "jammy-server-cloudimg-amd64.img" + ) + + +def test__resize_cloud_img_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess.run that raises an exception. + act: when _resize_cloud_img is called. + assert: ImageResizeError is raised. + """ + mock_run = MagicMock( + side_effect=subprocess.CalledProcessError( + returncode=1, cmd=[], output="", stderr="resize error" + ) + ) + monkeypatch.setattr( + subprocess, + "run", + mock_run, + ) + + with pytest.raises(ImageResizeError) as exc: + builder._resize_cloud_img(cloud_image_path=MagicMock()) + + assert "resize error" in str(exc.getrepr()) + + +def test__mount_nbd_partition(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched mock subprocess run. + act: when _mount_nbd_partition is called. + assert: subprocess run call is made. + """ + monkeypatch.setattr(subprocess, "run", (mock_run_call := MagicMock())) + + builder._mount_nbd_partition() + + mock_run_call.assert_called_once() + + +def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched process calls that fails. + act: when _mount_image_to_network_block_device is called. + assert: ImageMountError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=subprocess.CalledProcessError(1, [], "", "error mounting")), + ) + + with pytest.raises(ImageMountError) as exc: + builder._mount_image_to_network_block_device(cloud_image_path=MagicMock()) + + assert "error mounting" in str(exc.getrepr()) + + +def test__mount_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched mock process run calls and _mount_nbd_partition call. + act: when _mount_image_to_network_block_device is called. + assert: expected calls are made. + """ + monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(builder, "_mount_nbd_partition", (mount_mock := MagicMock())) + + builder._mount_image_to_network_block_device(cloud_image_path=MagicMock()) + + run_mock.assert_called_once() + mount_mock.assert_called_once() + + +def test__replace_mounted_resolv_conf(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched MOUNTED_RESOLV_CONF_PATH and shutil.copy call. + act: when _replace_mounted_resolv_conf. + assert: expected calls are made on the mocks. + """ + mock_mounted_resolv_conf_path = MagicMock() + mock_copy = MagicMock() + monkeypatch.setattr(builder, "MOUNTED_RESOLV_CONF_PATH", mock_mounted_resolv_conf_path) + monkeypatch.setattr(builder.shutil, "copy", mock_copy) + + builder._replace_mounted_resolv_conf() + + mock_mounted_resolv_conf_path.unlink.assert_called_once() + mock_copy.assert_called_once() + + +def test__resize_mount_partitions(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess calls that raises CalledProcessError. + act: when _resize_mount_partitions is called + assert: ResizePartitionError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=[None, subprocess.CalledProcessError(1, [], "", "resize error")]), + ) + + with pytest.raises(ResizePartitionError) as exc: + builder._resize_mount_partitions() + + assert "resize error" in str(exc.getrepr()) + + +def test__install_yq_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess.run function that raises an error. + act: when _install_yq is called. + assert: YQBuildError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=[None, subprocess.CalledProcessError(1, [], "", "Go build error.")]), + ) + + with pytest.raises(YQBuildError) as exc: + builder._install_yq() + + assert "Go build error" in str(exc.getrepr()) + + +def test__install_yq_already_exists(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched yq mocked path that already exists. + act: when _install_yq is called. + assert: Mock functions are called. + """ + monkeypatch.setattr(builder, "YQ_REPOSITORY_PATH", MagicMock(return_value=True)) + monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) + + builder._install_yq() + + run_mock.assert_called() + copy_mock.assert_called() + + +def test__install_yq(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched yq install mock functions. + act: when _install_yq is called. + assert: Mock functions are called. + """ + monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) + + builder._install_yq() + + run_mock.assert_called() + copy_mock.assert_called() + + +def test__create_python_symlinks(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched os.symlink function. + act: when _create_python_symlinks is called. + assert: the symlink function is called. + """ + mock_symlink_call = MagicMock() + monkeypatch.setattr(os, "symlink", mock_symlink_call) + + builder._create_python_symlinks() + + mock_symlink_call.assert_called() + + +def test__disable_unattended_upgrades_subprocess_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run function that raises SubprocessError. + act: when _disable_unattended_upgrades is called. + assert: the UnattendedUpgradeDisableError is raised. + """ + # Pylint thinks the testing mock patches are duplicate code (side effects are different). + # pylint: disable=duplicate-code + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=[ + *([None] * 7), + subprocess.SubprocessError("Failed to disable unattended upgrades"), + ] + ), + ) + + with pytest.raises(UnattendedUpgradeDisableError) as exc: + builder._disable_unattended_upgrades() + + assert "Failed to disable unattended upgrades" in str(exc.getrepr()) + + +def test__configure_system_users(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run calls that raises an exception. + act: when _configure_system_users is called. + assert: SystemUserConfigurationError is raised. + """ + monkeypatch.setattr(builder, "UBUNTU_HOME", MagicMock()) + monkeypatch.setattr( + builder.subprocess, + "run", + MagicMock(side_effect=[*([None] * 5), subprocess.SubprocessError("Failed to add group.")]), + ) + + with pytest.raises(SystemUserConfigurationError) as exc: + builder._configure_system_users() + + assert "Failed to add group." in str(exc.getrepr()) + + +def test__install_external_packages_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess.run that raises an error. + act: when _install_external_packages is called. + assert: ExternalPackageInstallError is raised. + """ + # The test mocks use similar codes. + monkeypatch.setattr( # pylint: disable=duplicate-code + subprocess, + "run", + MagicMock( + side_effect=[ + None, + subprocess.CalledProcessError(1, [], "", "Failed to clean npm cache."), + ] + ), + ) + + with pytest.raises(ExternalPackageInstallError) as exc: + builder._install_external_packages() + + assert "Failed to clean npm cache." in str(exc.getrepr()) + + +def test__install_external_packages(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched functions of _install_external_packages. + act: when _install_external_packages is called. + assert: The function exists without raising an error. + """ + monkeypatch.setattr(subprocess, "run", MagicMock()) + + assert builder._install_external_packages() is None + + +def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given subprocess run that raises CalledProcessError. + act: when _compress_image is called. + assert: ImageCompressError is raised. + """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=subprocess.CalledProcessError(1, [], "Compression error")), + ) + + with pytest.raises(ImageCompressError) as exc: + builder._compress_image(image=MagicMock()) + + assert "Compression error" in str(exc.getrepr()) + + +@pytest.mark.parametrize( + "patch_obj, sub_func, mock, expected_message", + [ + pytest.param( + builder, + "_resize_mount_partitions", + MagicMock(side_effect=ResizePartitionError("Partition resize failed")), + "Partition resize failed", + id="Partition resize failed", + ), + pytest.param( + builder, + "ChrootContextManager", + MagicMock(side_effect=ChrootBaseError("Failed to chroot into dir")), + "Failed to chroot into dir", + id="Failed to chroot into dir", + ), + pytest.param( + builder, + "_compress_image", + MagicMock(side_effect=ImageCompressError("Failed to compress image")), + "Failed to compress image", + id="Failed to compress image", + ), + ], +) +def test_build_image_error( + patch_obj: Any, + sub_func: str, + mock: MagicMock, + expected_message: str, + monkeypatch: pytest.MonkeyPatch, +): + """ + arrange: given a monkeypatched functions of build_image that raises exceptions. + act: when build_image is called. + assert: BuildImageError is raised. + """ + monkeypatch.setattr(builder, "_clean_build_state", MagicMock()) + monkeypatch.setattr(builder, "_download_cloud_image", MagicMock()) + monkeypatch.setattr(builder, "_resize_cloud_img", MagicMock()) + monkeypatch.setattr(builder, "_mount_image_to_network_block_device", MagicMock()) + monkeypatch.setattr(builder, "_resize_mount_partitions", MagicMock()) + monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) + monkeypatch.setattr(builder, "_install_yq", MagicMock()) + monkeypatch.setattr(builder, "ChrootContextManager", MagicMock()) + monkeypatch.setattr(builder.subprocess, "run", MagicMock()) + monkeypatch.setattr(builder, "_create_python_symlinks", MagicMock()) + monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) + monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) + monkeypatch.setattr(builder, "_install_external_packages", MagicMock()) + monkeypatch.setattr(builder, "_compress_image", MagicMock()) + monkeypatch.setattr(patch_obj, sub_func, mock) + + with pytest.raises(BuildImageError) as exc: + builder.build_image(config=MagicMock()) + + assert expected_message in str(exc.getrepr()) diff --git a/tests/unit/test_chroot.py b/tests/unit/test_chroot.py new file mode 100644 index 0000000..9335058 --- /dev/null +++ b/tests/unit/test_chroot.py @@ -0,0 +1,124 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for chroot module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +from unittest.mock import MagicMock + +import pytest + +from github_runner_image_builder.chroot import ( + ChrootContextManager, + MountError, + SyncError, + os, + subprocess, +) + + +def test_chroot_bind_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run call that fails. + act: when chroot context is entered. + assert: MountError is raised. + """ + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=subprocess.CalledProcessError( + returncode=1, cmd=[], output="", stderr="Failed to bind dirs" + ) + ), + ) + with pytest.raises(MountError) as exc: + with ChrootContextManager(chroot_path=MagicMock()): + pass + + assert "Failed to bind dirs" in str(exc.getrepr()) + + +def test_chroot_unmount_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run call that fails. + act: when chroot context is exited. + assert: MountError is raised. + """ + monkeypatch.setattr(os, "chroot", MagicMock()) + monkeypatch.setattr(os, "chdir", MagicMock()) + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=[ + *([None] * 5), + subprocess.CalledProcessError( + returncode=1, cmd=[], output="", stderr="Failed to unmount dirs" + ), + ] + ), + ) + with pytest.raises(MountError) as exc: + with ChrootContextManager(chroot_path=MagicMock()): + pass + + assert "Failed to unmount dirs" in str(exc.getrepr()) + + +def test_chroot_sync_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run call that fails. + act: when chroot context is exited. + assert: SyncError is raised. + """ + monkeypatch.setattr(os, "chroot", MagicMock()) + monkeypatch.setattr(os, "chdir", MagicMock()) + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=[ + *([None] * 3), + subprocess.CalledProcessError( + returncode=1, cmd=[], output="", stderr="Failed to sync dirs" + ), + ] + ), + ) + with pytest.raises(SyncError) as exc: + with ChrootContextManager(chroot_path=MagicMock()): + pass + + assert "Failed to sync dirs" in str(exc.getrepr()) + + +def test_chroot_umount_dev_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run call that fails. + act: when chroot context is exited. + assert: SyncError is raised. + """ + monkeypatch.setattr(os, "chroot", MagicMock()) + monkeypatch.setattr(os, "chdir", MagicMock()) + monkeypatch.setattr(os, "fchdir", MagicMock()) + monkeypatch.setattr(os, "close", MagicMock()) + monkeypatch.setattr( + subprocess, + "run", + MagicMock( + side_effect=[ + *([None] * 6), + subprocess.CalledProcessError( + returncode=1, cmd=[], output="", stderr="Failed to umount dev" + ), + ] + ), + ) + with pytest.raises(MountError) as exc: + with ChrootContextManager(chroot_path=MagicMock()): + pass + + assert "Failed to umount dev" in str(exc.getrepr()) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 0000000..11cbac5 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,230 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for cli module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +import itertools +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from github_runner_image_builder import cli +from github_runner_image_builder.cli import main + + +@pytest.fixture(scope="function", name="callback_path") +def callback_path_fixture(tmp_path: Path): + """The testing callback file path.""" + test_path = tmp_path / "test" + test_path.touch() + return test_path + + +@pytest.fixture(scope="function", name="build_image_inputs") +def build_image_inputs_fixture(callback_path: Path): + """Valid CLI inputs.""" + return { + "-i": "jammy", + "-c": "test-cloud-name", + "-n": "5", + "-p": str(callback_path), + "-o": "test-image", + } + + +def test__existing_path(tmp_path: Path): + """ + arrange: given a path that does not exist. + act: when _existing_path is called. + assert: ValueError is raised. + """ + not_exists_path = tmp_path / "non-existent" + with pytest.raises(ValueError) as exc: + cli._existing_path(str(not_exists_path)) + + assert f"Given path {not_exists_path} not found." in str(exc.getrepr()) + + +def test__install(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched builder.setup_builder function. + act: when _install is called. + assert: the mock function is called. + """ + monkeypatch.setattr(cli.builder, "setup_builder", (setup_mock := MagicMock())) + + cli._install() + + setup_mock.assert_called_once() + + +def test__get(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture): + """ + arrange: given a monkeypatched OpenstackManager.get_latest_image_id. + act: when _get is called. + assert: the image id is captured in stdout. + """ + openstack_manager_mock = MagicMock() + openstack_manager_mock.__enter__.return_value = (openstack_mock := MagicMock()) + openstack_mock.get_latest_image_id.return_value = "testing_id" + monkeypatch.setattr(cli, "OpenstackManager", MagicMock(return_value=openstack_manager_mock)) + + cli._get(cloud_name=MagicMock(), image_name=MagicMock()) + + res = capsys.readouterr() + assert "testing_id" in res.out + + +def test__build_and_upload(monkeypatch: pytest.MonkeyPatch, callback_path: Path): + """ + arrange: given a monkeypatched builder.setup_builder function. + act: when _build is called. + assert: the mock function is called. + """ + monkeypatch.setattr(cli.builder, "build_image", (builder_mock := MagicMock())) + monkeypatch.setattr( + cli, "OpenstackManager", MagicMock(return_value=(openstack_manager := MagicMock())) + ) + + cli._build_and_upload( + base="jammy", + callback_script_path=callback_path, + cloud_name=MagicMock(), + image_name=MagicMock(), + num_revisions=MagicMock(), + ) + + builder_mock.assert_called_once() + openstack_manager.__enter__.return_value.upload_image.assert_called_once() + + +@pytest.mark.parametrize( + "choice", + [ + pytest.param("", id="no choice"), + pytest.param("invalid", id="invalid choice"), + ], +) +def test_main_invalid_choice(monkeypatch: pytest.MonkeyPatch, choice: str): + """ + arrange: given invalid argument choice and mocked builder functions. + act: when main is called. + assert: SystemExit is raised and mocked builder functions are not called. + """ + monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) + monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + + with pytest.raises(SystemExit): + main([choice]) + + install_mock.assert_not_called() + get_mock.assert_not_called() + build_mock.assert_not_called() + + +def test_main_install(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given install argument and mocked builder functions. + act: when main is called. + assert: install builder mock function is called. + """ + monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) + monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + + main(["install"]) + + install_mock.assert_called() + get_mock.assert_not_called() + build_mock.assert_not_called() + + +def test_main_get(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given install argument and mocked builder functions. + act: when main is called. + assert: get mock function is called. + """ + monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) + monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + + main(["get", "-c", "test-cloud", "-o", "test-output-image-name"]) + + install_mock.assert_not_called() + get_mock.assert_called() + build_mock.assert_not_called() + + +@pytest.mark.parametrize( + "invalid_patch", + [ + pytest.param({"-i": ""}, id="no base-image"), + pytest.param({"-i": "test"}, id="invalid base-image"), + ], +) +def test_main_invalid_build_inputs( + monkeypatch: pytest.MonkeyPatch, + build_image_inputs: dict[str, str], + invalid_patch: dict[str, str], +): + """ + arrange: given invalid build arguments and mocked builder functions. + act: when main is called. + assert: SystemExit is raised. + """ + monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) + monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + build_image_inputs.update(invalid_patch) + inputs = list( + itertools.chain.from_iterable( + (flag, value) for (flag, value) in build_image_inputs.items() + ) + ) + + with pytest.raises(SystemExit): + main(["build", *inputs]) + + install_mock.assert_not_called() + get_mock.assert_not_called() + build_mock.assert_not_called() + + +@pytest.mark.parametrize( + "image", + [ + pytest.param("jammy", id="jammy"), + pytest.param("22.04", id="jammy tag"), + pytest.param("noble", id="noble"), + pytest.param("24.04", id="noble tag"), + ], +) +def test_main_base_image( + monkeypatch: pytest.MonkeyPatch, image: str, build_image_inputs: dict[str, str] +): + """ + arrange: given invalid base_image argument and mocked builder functions. + act: when main is called. + assert: build image is called. + """ + monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) + monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + build_image_inputs.update({"-i": image}) + inputs = list( + itertools.chain.from_iterable( + (flag, value) for (flag, value) in build_image_inputs.items() + ) + ) + + main(["build", *inputs]) + + install_mock.assert_not_called() + get_mock.assert_not_called() + build_mock.assert_called() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..40958fd --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,96 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for state module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +import platform + +import pytest + +from github_runner_image_builder.config import ( + Arch, + BaseImage, + UnsupportedArchitectureError, + get_supported_arch, +) + + +@pytest.mark.parametrize( + "arch", + [ + pytest.param("ppc64le", id="ppc64le"), + pytest.param("mips", id="mips"), + pytest.param("s390x", id="s390x"), + pytest.param("testing", id="testing"), + ], +) +def test_get_supported_arch_unsupported_arch(arch: str, monkeypatch: pytest.MonkeyPatch): + """ + arrange: given architectures that are not supported by the app. + act: when get_supported_arch is called. + assert: UnsupportedArchitectureError is raised + """ + monkeypatch.setattr(platform, "machine", lambda: arch) + + with pytest.raises(UnsupportedArchitectureError) as exc: + get_supported_arch() + + assert arch in str(exc.getrepr()) + + +@pytest.mark.parametrize( + "arch, expected_arch", + [ + pytest.param("aarch64", Arch.ARM64, id="aarch64"), + pytest.param("arm64", Arch.ARM64, id="aarch64"), + pytest.param("x86_64", Arch.X64, id="amd64"), + ], +) +def test_get_supported_arch(arch: str, expected_arch: Arch, monkeypatch: pytest.MonkeyPatch): + """ + arrange: given architectures that is supported by the app. + act: when get_supported_arch is called. + assert: expected architecture is returned. + """ + monkeypatch.setattr(platform, "machine", lambda: arch) + + assert get_supported_arch() == expected_arch + + +@pytest.mark.parametrize( + "image", + [ + pytest.param("dingo", id="dingo"), + pytest.param("focal", id="focal"), + pytest.param("firefox", id="firefox"), + ], +) +def test_base_image_invalid(image: str): + """ + arrange: given invalid or unsupported base image names as config value. + act: when BaseImage.from_str is called. + assert: ValueError is raised. + """ + with pytest.raises(ValueError) as exc: + BaseImage.from_str(image) + + assert image in str(exc) + + +@pytest.mark.parametrize( + "image, expected_base_image", + [ + pytest.param("jammy", BaseImage.JAMMY, id="jammy"), + pytest.param("22.04", BaseImage.JAMMY, id="22.04"), + ], +) +def test_base_image(image: str, expected_base_image: BaseImage): + """ + arrange: given supported image name or tag as config value. + act: when BaseImage.from_str is called. + assert: expected base image is returned. + """ + assert BaseImage.from_str(image) == expected_base_image diff --git a/tests/unit/test_upload.py b/tests/unit/test_upload.py new file mode 100644 index 0000000..5e48663 --- /dev/null +++ b/tests/unit/test_upload.py @@ -0,0 +1,227 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for builder module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +from unittest.mock import MagicMock + +import pytest + +from github_runner_image_builder.upload import ( + GetImageError, + Image, + OpenstackConnectionError, + OpenstackManager, + UnauthorizedError, + UploadImageError, + openstack, +) +from tests.unit.factories import MockOpenstackImageFactory + + +@pytest.fixture(name="connection") +def mocked_openstack_connection_fixture(): + """Fixture for Openstack connection context manager mock instance.""" + connection_mock = MagicMock() + return connection_mock + + +@pytest.fixture(name="manager") +def openstack_manager_mock_fixture(monkeypatch: pytest.MonkeyPatch, connection: MagicMock): + """Fixture for OpenstackManager.""" + monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connection)) + return OpenstackManager(cloud_name="test") + + +def test_openstack_manager_context( + monkeypatch: pytest.MonkeyPatch, connection: MagicMock, manager: OpenstackManager +): + """ + arrange: given a monkeypatched openstack connection. + act: when openstck manager context is entered and exited. + assert: connection is closed. + """ + monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connection)) + + with manager: + pass + + connection.close.assert_called_once() + + +def test___init__error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched openstack authorize function that raises an exception. + act: when OpenstackManager is initialized. + assert: UnauthorizedError is raised. + """ + connect_mock = MagicMock() + connect_mock.__enter__.return_value = (connection_mock := MagicMock()) + connection_mock.authorize.side_effect = openstack.exceptions.HttpException + monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connect_mock)) + + with pytest.raises(UnauthorizedError) as exc: + OpenstackManager(cloud_name="tests") + + assert "Unauthorized credentials." in str(exc.getrepr()) + + +def test__get_images_by_latest_error(connection: MagicMock, manager: OpenstackManager): + """ + arrange: given a mocked openstack connection that returns images in non-sorted order. + act: when _get_images_by_latest is called. + assert: the images are returned in sorted order by creation date. + """ + connection.search_images.side_effect = openstack.exceptions.OpenStackCloudException( + "Network error" + ) + + with pytest.raises(OpenstackConnectionError) as err: + manager._get_images_by_latest(image_name=MagicMock) + + assert "Network error" in str(err.getrepr()) + + +def test__get_images_by_latest(connection: MagicMock, manager: OpenstackManager): + """ + arrange: given a mocked openstack connection that returns images in non-sorted order. + act: when _get_images_by_latest is called. + assert: the images are returned in sorted order by creation date. + """ + connection.search_images.return_value = [ + (first := MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z")), + (third := MockOpenstackImageFactory(id="3", created_at="2024-03-03T00:00:00Z")), + (second := MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z")), + ] + + assert manager._get_images_by_latest(image_name=MagicMock) == [third, second, first] + + +def test__prune_old_images_error( + caplog: pytest.LogCaptureFixture, + connection: MagicMock, + manager: OpenstackManager, +): + """ + arrange: given a mocked delete function that raises an exception. + act: when _prune_old_images is called. + assert: failure to delete is logged. + """ + connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + connection.delete_image.side_effect = openstack.exceptions.OpenStackCloudException( + "Delete error" + ) + + manager._prune_old_images(image_name=MagicMock(), num_revisions=0) + + assert all("Failed to prune old image" in log for log in caplog.messages) + + +def test__prune_old_images_fail( + caplog: pytest.LogCaptureFixture, connection: MagicMock, manager: OpenstackManager +): + """ + arrange: given a mocked delete function that returns false. + act: when _prune_old_images is called. + assert: failure to delete is logged. + """ + connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + connection.delete_image.return_value = False + + manager._prune_old_images(image_name=MagicMock(), num_revisions=0) + + assert all("Failed to delete old image" in log for log in caplog.messages) + + +def test__prune_old_images(connection: MagicMock, manager: OpenstackManager): + """ + arrange: given a mocked delete function that returns true. + act: when _prune_old_images is called. + assert: delete mock is called. + """ + connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + connection.delete_image.return_value = True + + manager._prune_old_images(image_name=MagicMock(), num_revisions=0) + + assert connection.delete_image.call_count == 2 + + +def test_upload_image_error(connection: MagicMock, manager: OpenstackManager): + """ + arrange: given a mocked openstack create_image function that raises an exception. + act: when upload_image is called. + assert: UploadImageError is raised. + """ + connection.create_image.side_effect = openstack.exceptions.OpenStackCloudException( + "Resource capacity exceeded." + ) + + with pytest.raises(UploadImageError) as exc: + manager.upload_image(config=MagicMock()) + + assert "Resource capacity exceeded." in str(exc.getrepr()) + + +def test_upload_image(connection: MagicMock, manager: OpenstackManager): + """ + arrange: given a mocked openstack create_image function that raises an exception. + act: when upload_image is called. + assert: UploadImageError is raised. + """ + connection.create_image.return_value = MockOpenstackImageFactory(id="1") + + assert manager.upload_image(config=MagicMock()) == "1" + + +def test_get_latest_image_id_error(manager: OpenstackManager): + """ + arrange: given a mocked _get_images_by_latest function that raises an exception. + act: when get_latest_image_id is called. + assert: GetImageError is raised. + """ + manager._get_images_by_latest = MagicMock(side_effect=OpenstackConnectionError("Unauthorized")) + + with pytest.raises(GetImageError) as exc: + manager.get_latest_image_id(image_name=MagicMock()) + + assert "Unauthorized" in str(exc.getrepr()) + + +@pytest.mark.parametrize( + "images, expected_id", + [ + pytest.param([], None, id="No images"), + pytest.param( + [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ], + "1", + id="Multiple images", + ), + ], +) +def test_get_latest_image_id( + manager: OpenstackManager, images: list[Image], expected_id: str | None +): + """ + arrange: given a mocked _get_images_by_latest function that returns openstack images. + act: when get_latest_image_id is called. + assert: GetImageError is raised. + """ + manager._get_images_by_latest = MagicMock(return_value=images) + + assert manager.get_latest_image_id(image_name=MagicMock()) == expected_id diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..fd33270 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,78 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for utils module.""" + +import logging +from unittest.mock import MagicMock + +import pytest + +from github_runner_image_builder.utils import retry + + +def test_retry_with_logger(): + """ + arrange: given a function that raises exception decorated with retry. + act: when the function is called. + assert: the function is retried and raises eventually. + """ + counter = MagicMock() + num_tries = 2 + logger = logging.getLogger("test") + + with pytest.raises(ValueError): + + @retry( + exception=ValueError, + tries=num_tries, + delay=0, + max_delay=0, + backoff=1, + local_logger=logger, + ) + def decorated_func(): + """A test function that is being decorated. + + Raises: + ValueError: always. + """ + counter() + raise ValueError + + decorated_func() + + assert counter.call_count == num_tries + + +def test_retry_no_logger(): + """ + arrange: given a function that raises exception decorated with retry. + act: when the function is called. + assert: the function is retried and raises eventually. + """ + counter = MagicMock() + num_tries = 2 + + with pytest.raises(ValueError): + + @retry( + exception=ValueError, + tries=num_tries, + delay=0, + max_delay=None, + backoff=1, + local_logger=None, + ) + def decorated_func(): + """A test function that is being decorated. + + Raises: + ValueError: always. + """ + counter() + raise ValueError + + decorated_func() + + assert counter.call_count == num_tries diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ab4c184 --- /dev/null +++ b/tox.ini @@ -0,0 +1,119 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +[tox] +skipsdist=True +skip_missing_interpreters = True +envlist = lint, unit, static, coverage-report + +[vars] +src_path = {toxinidir}/src/ +tst_path = {toxinidir}/tests/ +all_path = {[vars]src_path} {[vars]tst_path} + +[testenv] +basepython = python3.10 +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 +passenv = + PYTHONPATH + +[testenv:fmt] +description = Apply coding style standards to code +deps = + black + isort +commands = + isort {[vars]all_path} + black {[vars]all_path} + +[testenv:lint] +description = Check code against coding style standards +deps = + black + codespell + flake8<6.0.0 + flake8-builtins + flake8-copyright<6.0.0 + flake8-docstrings>=1.6.0 + flake8-docstrings-complete>=1.0.3 + flake8-test-docs>=1.0 + isort + mypy + pep8-naming + pydocstyle>=2.10 + pylint + pyproject-flake8<6.0.0 + pytest + pytest-asyncio + requests + types-PyYAML + types-requests + -r{toxinidir}/requirements.txt + -r{[vars]tst_path}unit/requirements.txt + -r{[vars]tst_path}integration/requirements.txt +commands = + pydocstyle {[vars]src_path} + codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ + --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ + --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg + # pflake8 wrapper supports config from pyproject.toml + pflake8 {[vars]all_path} --ignore=W503 + isort --check-only --diff {[vars]all_path} + black --check --diff {[vars]all_path} + mypy {[vars]all_path} + pylint {[vars]all_path} + +[testenv:unit] +description = Run unit tests +deps = + coverage[toml] + pytest + -r{toxinidir}/requirements.txt + -r{[vars]tst_path}unit/requirements.txt +commands = + coverage run --source={[vars]src_path} \ + -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} + # Omit main entrypoint file + coverage report --omit={[vars]src_path}/github_runner_image_builder/__main__.py + +[testenv:coverage-report] +description = Create test coverage report +deps = + coverage[toml] + pytest + -r{toxinidir}/requirements.txt + -r{[vars]tst_path}unit/requirements.txt +commands = + # Omit main entrypoint file + coverage report --omit={[vars]src_path}/github_runner_image_builder/__main__.py + +[testenv:static] +description = Run static analysis tests +deps = + bandit[toml] + -r{toxinidir}/requirements.txt +commands = + bandit -c {toxinidir}/pyproject.toml -r {[vars]src_path} {[vars]tst_path} + +[testenv:integration] +description = Run integration tests +deps = + pytest + pytest-asyncio + -r{toxinidir}/requirements.txt + -r{[vars]tst_path}integration/requirements.txt +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + +[testenv:src-docs] +allowlist_externals=sh +description = Generate documentation for src +deps = + lazydocs + -r{toxinidir}/requirements.txt +commands = + ; can't run lazydocs directly due to needing to run it on src/* which produces an invocation error in tox + sh generate-src-docs.sh From aa412146e6455c2f6c75bcb06a82a659eae24b5f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 29 May 2024 03:41:08 +0000 Subject: [PATCH 03/63] small change --- src/github_runner_image_builder/builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 5d500bc..6521c47 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -95,6 +95,8 @@ "gh", ] +print("ehllo") + def _install_dependencies() -> None: """Install required dependencies to run qemu image build. From 703eb4daf50e62d386c86400efe8aa895da98661 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 29 May 2024 03:45:28 +0000 Subject: [PATCH 04/63] small change --- src/github_runner_image_builder/builder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 6521c47..5d500bc 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -95,8 +95,6 @@ "gh", ] -print("ehllo") - def _install_dependencies() -> None: """Install required dependencies to run qemu image build. From 687563e62fbc1f086de891e6e08208f3de73aead Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:22:01 +0000 Subject: [PATCH 05/63] comment fixes --- .github/workflows/integration_test.yaml | 23 +- pyproject.toml | 15 +- requirements.txt | 1 - src/github_runner_image_builder/__main__.py | 4 +- src/github_runner_image_builder/builder.py | 303 +++++++++----------- src/github_runner_image_builder/cli.py | 242 ++++++++-------- src/github_runner_image_builder/config.py | 12 +- src/github_runner_image_builder/errors.py | 10 +- src/github_runner_image_builder/store.py | 115 ++++++++ src/github_runner_image_builder/upload.py | 159 ---------- tests/integration/helpers.py | 6 + tests/unit/test_builder.py | 107 +++---- tests/unit/test_cli.py | 143 ++++----- tests/unit/test_store.py | 210 ++++++++++++++ tests/unit/test_upload.py | 227 --------------- 15 files changed, 737 insertions(+), 840 deletions(-) create mode 100644 src/github_runner_image_builder/store.py delete mode 100644 src/github_runner_image_builder/upload.py create mode 100644 tests/unit/test_store.py delete mode 100644 tests/unit/test_upload.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 58069e5..8430323 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -8,9 +8,28 @@ concurrency: cancel-in-progress: true jobs: - integration-tests: + integration-tests-arm: name: Integration test - runs-on: [self-hosted, stg-private-endpoint] + runs-on: [self-hosted, ARM64] + strategy: + matrix: + image: [jammy, noble] + steps: + - uses: actions/checkout@v3 + - uses: canonical/setup-lxd@v0.1.1 + - name: Setup python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.py }} + # need to run in sudo mode due to chroot + - name: Install tox + run: sudo python -m pip install tox-gh + - name: Run integration tests + run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} + + integration-tests-amd: + name: Integration test + runs-on: [self-hosted, X64] strategy: matrix: image: [jammy, noble] diff --git a/pyproject.toml b/pyproject.toml index 87cfdd2..828c81a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,14 @@ [project] name = "github-runner-image-builder" -version = "0.0.1" -authors = [{ name = "Yanks Yoon", email = "yangsoo.yoon@canonical.com" }] +version = "0.1" +authors = [{ name = "Canonical IS DevOps", email = "is-devops-team@canonical.com" }] description = "A github runner image builder package" readme = "README.md" requires-python = ">=3.10" classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Apache License", "Operating System :: OS Independent", ] dynamic = ["dependencies"] @@ -18,8 +18,8 @@ dynamic = ["dependencies"] dependencies = { file = ["requirements.txt"] } [project.urls] -Homepage = "https://github.com/canonical/github-runner-image-builder-snap" -Issues = "https://github.com/canonical/github-runner-image-builder-snap/issues" +Homepage = "https://github.com/canonical/github-runner-image-builder" +Issues = "https://github.com/canonical/github-runner-image-builder/issues" [project.scripts] github-runner-image-builder = "github_runner_image_builder.cli:main" @@ -41,7 +41,6 @@ fail_under = 100 show_missing = true [tool.pytest.ini_options] -minversion = "6.0" log_cli_level = "INFO" # Formatting tools configuration @@ -63,8 +62,8 @@ select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503 because using black creates errors with this # Ignore D107 Missing docstring in __init__ ignore = ["W503", "D107"] -# D100, D101, D102, D103, D104, DCO020, DCO030: Ignore docstring style issues in tests -per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212,DCO020,DCO030"] +# D100, D101, D102, D103, D104: Ignore docstring style issues in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212"] docstring-convention = "google" # Check for properly formatted copyright header in each file copyright-check = "True" diff --git a/requirements.txt b/requirements.txt index 799c087..721f044 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ -typing-extensions==4.11.0 pyyaml==6.0.1 openstacksdk==3.1.0 diff --git a/src/github_runner_image_builder/__main__.py b/src/github_runner_image_builder/__main__.py index 9016dde..bf8b2c5 100644 --- a/src/github_runner_image_builder/__main__.py +++ b/src/github_runner_image_builder/__main__.py @@ -3,9 +3,7 @@ """Main entrypoint for github-runner-image-builder.""" -import sys - from github_runner_image_builder.cli import main if __name__ == "__main__": - sys.exit(main()) + main() diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 5d500bc..966b600 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -3,29 +3,24 @@ """Module for interacting with qemu image builder.""" -import dataclasses import logging -import os import shutil # Ignore B404:blacklist since all subprocesses are run with predefined executables. import subprocess # nosec -import sys import urllib.error import urllib.request -from contextlib import redirect_stdout from pathlib import Path from typing import Literal from github_runner_image_builder.chroot import ChrootBaseError, ChrootContextManager from github_runner_image_builder.config import IMAGE_OUTPUT_PATH, Arch, BaseImage from github_runner_image_builder.errors import ( + BaseImageDownloadError, BuilderSetupError, BuildImageError, CleanBuildStateError, - CloudImageDownloadError, DependencyInstallError, - ExternalPackageInstallError, ImageBuilderBaseError, ImageCompressError, ImageMountError, @@ -35,13 +30,14 @@ SystemUserConfigurationError, UnattendedUpgradeDisableError, UnsupportedArchitectureError, + YarnInstallError, YQBuildError, ) from github_runner_image_builder.utils import retry logger = logging.getLogger(__name__) -SupportedCloudImageArch = Literal["amd64", "arm64"] +SupportedBaseImageArch = Literal["amd64", "arm64"] APT_DEPENDENCIES = [ "qemu-utils", # used for qemu utilities tools to build and resize image @@ -54,7 +50,7 @@ # Constants for mounting images IMAGE_MOUNT_DIR = Path("/mnt/ubuntu-image/") NETWORK_BLOCK_DEVICE_PATH = Path("/dev/nbd0") -NETWORK_BLOCK_DEVICE_PARTITION_PATH = Path("/dev/nbd0p1") +NETWORK_BLOCK_DEVICE_PARTITION_PATH = Path(f"{NETWORK_BLOCK_DEVICE_PATH}p1") # Constants for building image # This amount is the smallest increase that caters for the installations within this image. @@ -86,16 +82,30 @@ MOUNTED_YQ_BIN_PATH = IMAGE_MOUNT_DIR / "usr/bin/yq" IMAGE_DEFAULT_APT_PACKAGES = [ "docker.io", + "gh", + "jq", "npm", "python3-pip", + "python-is-python3", "shellcheck", - "jq", - "wget", "unzip", - "gh", + "wget", ] +def initialize() -> None: + """Configure the host machine to build images. + + Raises: + BuilderSetupError: If there was an error setting up the host device for building images. + """ + try: + _install_dependencies() + _enable_network_block_device() + except ImageBuilderBaseError as exc: + raise BuilderSetupError from exc + + def _install_dependencies() -> None: """Install required dependencies to run qemu image build. @@ -103,24 +113,24 @@ def _install_dependencies() -> None: DependencyInstallError: If there was an error installing apt packages. """ try: - subprocess.run( - ["/usr/bin/apt-get", "update", "-y"], check=True, timeout=30 * 60 + subprocess.check_output( + ["/usr/bin/apt-get", "update", "-y"], encoding="utf-8", timeout=30 * 60 ) # nosec: B603 - subprocess.run( + subprocess.check_output( ["/usr/bin/apt-get", "install", "-y", "--no-install-recommends", *APT_DEPENDENCIES], - check=True, + encoding="utf-8", timeout=30 * 60, ) # nosec: B603 - subprocess.run( + subprocess.check_output( ["/usr/bin/snap", "install", SNAP_GO, "--classic"], - check=True, + encoding="utf-8", timeout=30 * 60, ) # nosec: B603 except subprocess.CalledProcessError as exc: raise DependencyInstallError from exc -def _enable_nbd() -> None: +def _enable_network_block_device() -> None: """Enable network block device module to mount and build chrooted image. Raises: @@ -132,44 +142,69 @@ def _enable_nbd() -> None: raise NetworkBlockDeviceError from exc -def setup_builder() -> None: - """Configure the host machine to build images. +def build_image(arch: Arch, base_image: BaseImage) -> None: + """Build and save the image locally. + + Args: + arch: The CPU architecture to build the image for. + base_image: The ubuntu image to use as build base. Raises: - BuilderSetupError: If there was an error setting up the host device for building images. + BuildImageError: If there was an error building the image. """ + IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) try: - _install_dependencies() - _enable_nbd() + logger.info("Cleaning build state.") + _clean_build_state() + logger.info("Downloading base image.") + base_image_path = _download_base_image(arch=arch, base_image=base_image) + logger.info("Resizing base image.") + _resize_image(image_path=base_image_path) + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() + logger.info("Mounting network block device.") + _mount_image_to_network_block_device(image_path=base_image_path) + logger.info("Resizing partitions.") + _resize_mount_partitions() + logger.info("Installing YQ from source.") + _install_yq() except ImageBuilderBaseError as exc: - raise BuilderSetupError from exc - - -def _get_supported_runner_arch(arch: Arch) -> SupportedCloudImageArch: - """Validate and return supported runner architecture. - - The supported runner architecture takes in arch value from Github supported - architecture and outputs architectures supported by ubuntu cloud images. - See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-\ - self-hosted-runners/about-self-hosted-runners#architectures - and https://cloud-images.ubuntu.com/jammy/current/ - - Args: - arch: The compute architecture to check support for. + raise BuildImageError from exc - Raises: - UnsupportedArchitectureError: If an unsupported architecture was passed. + try: + logger.info("Setting up chroot environment.") + with ChrootContextManager(IMAGE_MOUNT_DIR): + # operator_libs_linux apt package uses dpkg -l and that does not work well with + # chroot env, hence use subprocess run. + subprocess.run( + ["/usr/bin/apt-get", "update", "-y"], + check=True, + timeout=60 * 10, + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) # nosec: B603 + subprocess.run( # nosec: B603 + [ + "/usr/bin/apt-get", + "install", + "-y", + "--no-install-recommends", + *IMAGE_DEFAULT_APT_PACKAGES, + ], + check=True, + timeout=60 * 20, + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + _disable_unattended_upgrades() + _configure_system_users() + _install_yarn() + except ChrootBaseError as exc: + raise BuildImageError from exc - Returns: - The supported architecture. - """ - match arch: - case Arch.X64: - return "amd64" - case Arch.ARM64: - return "arm64" - case _: - raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") + try: + logger.info("Compressing image.") + _compress_image(base_image_path) + except ImageBuilderBaseError as exc: + raise BuildImageError from exc def _clean_build_state() -> None: @@ -180,7 +215,6 @@ def _clean_build_state() -> None: """ # The commands will fail if artefacts do not exist and hence there is no need to check the # output of subprocess runs. - IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) try: subprocess.run( ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False @@ -214,23 +248,23 @@ def _clean_build_state() -> None: raise CleanBuildStateError from exc -def _download_cloud_image(arch: Arch, base_image: BaseImage) -> Path: - """Download the cloud image from cloud-images.ubuntu.com. +def _download_base_image(arch: Arch, base_image: BaseImage) -> Path: + """Download the base image from cloud-images.ubuntu.com. Args: - arch: The cloud image architecture to download. + arch: The base image architecture to download. base_image: The ubuntu base image OS to download. Returns: - The downloaded cloud image path. + The downloaded image path. Raises: - CloudImageDownloadError: If there was an error downloading the image. + BaseImageDownloadError: If there was an error downloading the image. """ try: bin_arch = _get_supported_runner_arch(arch) except UnsupportedArchitectureError as exc: - raise CloudImageDownloadError from exc + raise BaseImageDownloadError from exc try: # The ubuntu-cloud-images is a trusted source @@ -244,21 +278,48 @@ def _download_cloud_image(arch: Arch, base_image: BaseImage) -> Path: ) return Path(image_path) except urllib.error.URLError as exc: - raise CloudImageDownloadError from exc + raise BaseImageDownloadError from exc -def _resize_cloud_img(cloud_image_path: Path) -> None: - """Resize cloud image to allow space for dependency installations. +def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: + """Validate and return supported runner architecture. + + The supported runner architecture takes in arch value from Github supported + architecture and outputs architectures supported by ubuntu cloud images. + See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-\ + self-hosted-runners/about-self-hosted-runners#architectures + and https://cloud-images.ubuntu.com/jammy/current/ Args: - cloud_image_path: The target cloud image file to resize. + arch: The compute architecture to check support for. + + Raises: + UnsupportedArchitectureError: If an unsupported architecture was passed. + + Returns: + The supported architecture. + """ + match arch: + case Arch.X64: + return "amd64" + case Arch.ARM64: + return "arm64" + case _: + raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") + + +def _resize_image(image_path: Path) -> None: + """Resize image to allow space for dependency installations. + + Args: + image_path: The target image file to resize. Raises: ImageResizeError: If there was an error resizing the image. """ try: subprocess.run( # nosec: B603 - ["/usr/bin/qemu-img", "resize", str(cloud_image_path), RESIZE_AMOUNT], + ["/usr/bin/qemu-img", "resize", str(image_path), RESIZE_AMOUNT], check=True, timeout=60, ) @@ -266,8 +327,14 @@ def _resize_cloud_img(cloud_image_path: Path) -> None: raise ImageResizeError from exc +def _replace_mounted_resolv_conf() -> None: + """Replace resolv.conf to host resolv.conf to allow networking.""" + MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) + shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) + + @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) -def _mount_nbd_partition() -> None: +def _mount_network_block_device_partition() -> None: """Mount the network block device partition.""" subprocess.run( # nosec: B603 [ @@ -282,32 +349,26 @@ def _mount_nbd_partition() -> None: ) -def _mount_image_to_network_block_device(cloud_image_path: Path) -> None: +def _mount_image_to_network_block_device(image_path: Path) -> None: """Mount the image to network block device in preparation for chroot. Args: - cloud_image_path: The target cloud image file to mount. + image_path: The target image file to mount. Raises: ImageMountError: If there was an error mounting the image to network block device. """ try: subprocess.run( # nosec: B603 - ["/usr/bin/qemu-nbd", f"--connect={NETWORK_BLOCK_DEVICE_PATH}", str(cloud_image_path)], + ["/usr/bin/qemu-nbd", f"--connect={NETWORK_BLOCK_DEVICE_PATH}", str(image_path)], check=True, timeout=60, ) - _mount_nbd_partition() + _mount_network_block_device_partition() except subprocess.CalledProcessError as exc: raise ImageMountError from exc -def _replace_mounted_resolv_conf() -> None: - """Replace resolv.conf to host resolv.conf to allow networking.""" - MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) - shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) - - def _resize_mount_partitions() -> None: """Resize the block partition to fill available space. @@ -356,11 +417,6 @@ def _install_yq() -> None: raise YQBuildError from exc -def _create_python_symlinks() -> None: - """Create python3 symlinks.""" - os.symlink(DEFAULT_PYTHON_PATH, SYM_LINK_PYTHON_PATH) - - def _disable_unattended_upgrades() -> None: """Disable unatteneded upgrades to prevent apt locks. @@ -432,13 +488,11 @@ def _configure_system_users() -> None: raise SystemUserConfigurationError from exc -def _install_external_packages() -> None: - """Install packages outside of apt. - - Installs yarn. +def _install_yarn() -> None: + """Install yarn using NPM. Raises: - ExternalPackageInstallError: If there was an error installing external package. + YarnInstallError: If there was an error installing external package. """ try: # 2024/04/26 There's a potential security risk here, npm is subject to toolchain attacks. @@ -449,12 +503,12 @@ def _install_external_packages() -> None: ["/usr/bin/npm", "cache", "clean", "--force"], check=True, timeout=60 ) # nosec: B603 except subprocess.SubprocessError as exc: - raise ExternalPackageInstallError from exc + raise YarnInstallError from exc @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _compress_image(image: Path) -> None: - """Compress the cloud image. + """Compress the image. Args: image: The image to compress. @@ -470,84 +524,3 @@ def _compress_image(image: Path) -> None: ) except subprocess.CalledProcessError as exc: raise ImageCompressError from exc - - -@dataclasses.dataclass -class BuildImageConfig: - """Configuration for building the image. - - Attributes: - arch: The CPU architecture to build the image for. - base_image: The ubuntu image to use as build base. - """ - - arch: Arch - base_image: BaseImage - - -def build_image(config: BuildImageConfig) -> None: - """Build and save the image locally. - - Args: - config: The configuration values to build the image with. - - Raises: - BuildImageError: If there was an error building the image. - """ - logger.info("Clean build state.") - with redirect_stdout(sys.stderr): - try: - _clean_build_state() - logger.info("Downloading cloud image.") - cloud_image_path = _download_cloud_image( - arch=config.arch, base_image=config.base_image - ) - logger.info("Resizing cloud image.") - _resize_cloud_img(cloud_image_path=cloud_image_path) - logger.info("Mounting network block device.") - _mount_image_to_network_block_device(cloud_image_path=cloud_image_path) - logger.info("Replacing resolv.conf.") - _replace_mounted_resolv_conf() - logger.info("Resizing partitions.") - _resize_mount_partitions() - logger.info("Building YQ from source.") - _install_yq() - except ImageBuilderBaseError as exc: - raise BuildImageError from exc - - try: - logger.info("Setting up chroot environment.") - with ChrootContextManager(IMAGE_MOUNT_DIR): - # operator_libs_linux apt package uses dpkg -l and that does not work well with - # chroot env, hence use subprocess run. - subprocess.run( - ["/usr/bin/apt-get", "update", "-y"], - check=True, - timeout=60 * 10, - env={"DEBIAN_FRONTEND": "noninteractive"}, - ) # nosec: B603 - subprocess.run( # nosec: B603 - [ - "/usr/bin/apt-get", - "install", - "-y", - "--no-install-recommends", - *IMAGE_DEFAULT_APT_PACKAGES, - ], - check=True, - timeout=60 * 20, - env={"DEBIAN_FRONTEND": "noninteractive"}, - ) - _create_python_symlinks() - _disable_unattended_upgrades() - _configure_system_users() - _install_external_packages() - except (ImageBuilderBaseError, ChrootBaseError) as exc: - raise BuildImageError from exc - - try: - _clean_build_state() - logger.info("Compressing image") - _compress_image(cloud_image_path) - except ImageBuilderBaseError as exc: - raise BuildImageError from exc diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index c86a0c3..536a090 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -11,84 +11,17 @@ from pathlib import Path from typing import cast -from github_runner_image_builder import builder -from github_runner_image_builder.builder import BuildImageConfig +from github_runner_image_builder import builder, store from github_runner_image_builder.config import ( + ACTION_INIT, + ACTION_LATEST_BUILD_ID, + ACTION_RUN, IMAGE_OUTPUT_PATH, LTS_IMAGE_VERSION_TAG_MAP, ActionsNamespace, BaseImage, get_supported_arch, ) -from github_runner_image_builder.upload import OpenstackManager, UploadImageConfig - - -def _existing_path(value: str) -> Path: - """Check the path exists. - - Args: - value: The path string. - - Raises: - ValueError: If the path does not exist. - - Returns: - Path that exists. - """ - path = Path(value) - if not path.exists(): - raise ValueError(f"Given path {value} not found.") - return path - - -def _install() -> None: - """Install builder.""" - builder.setup_builder() - - -def _get(cloud_name: str, image_name: str) -> None: - """Get latest built image from OpenStack. - - Args: - cloud_name: The Openstack cloud to upload the image to. - image_name: The image name to upload as. - """ - with OpenstackManager(cloud_name=cloud_name) as manager: - sys.stdout.write(manager.get_latest_image_id(image_name=image_name)) - - -def _build_and_upload( - base: str, - callback_script_path: Path, - cloud_name: str, - image_name: str, - num_revisions: int, -) -> None: - """Build and upload image. - - Args: - base: Ubuntu image base. - callback_script_path: Path to bash script to call after image upload. - cloud_name: The Openstack cloud to upload the image to. - image_name: The image name to upload as. - num_revisions: Number of image revisions to keep before deletion. - """ - arch = get_supported_arch() - base_image = BaseImage.from_str(base) - build_config = BuildImageConfig(arch=arch, base_image=base_image) - builder.build_image(config=build_config) - with OpenstackManager(cloud_name=cloud_name) as manager: - image_id = manager.upload_image( - config=UploadImageConfig( - arch=arch, - base=base_image, - image_name=image_name, - num_revisions=num_revisions, - src_path=IMAGE_OUTPUT_PATH, - ) - ) - # The callback script is a user trusted script. - subprocess.check_call(["/bin/bash", str(callback_script_path), image_id]) # nosec: B603 def main(args: list[str] | None = None) -> None: @@ -111,29 +44,40 @@ def main(args: list[str] | None = None) -> None: dest="action", required=True, ) - subparsers.add_parser("install") - get_parser = subparsers.add_parser("get") - get_parser.add_argument( - "-c", - "--cloud-name", + subparsers.add_parser(ACTION_INIT) + get_latest_id_parser = subparsers.add_parser( + ACTION_LATEST_BUILD_ID, description="Fetch the latest ID of the built image." + ) + run_parser = subparsers.add_parser(ACTION_RUN, description="Build the image.") + get_latest_id_parser.add_argument( dest="cloud_name", - required=True, help=( "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " "of the following order: current directory, ~/.config/openstack, /etc/openstack." ), + type=non_empty_string, ) - get_parser.add_argument( - "-o", - "--output-image-name", + get_latest_id_parser.add_argument( dest="image_name", - required=True, help="The image name uploaded to Openstack.", + type=non_empty_string, + ) + run_parser.add_argument( + dest="cloud_name", + help=( + "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " + "of the following order: current directory, ~/.config/openstack, /etc/openstack." + ), + type=non_empty_string, ) - build_parser = subparsers.add_parser("build") - build_parser.add_argument( - "-i", - "--image-base", + run_parser.add_argument( + dest="image_name", + help="The image name to upload to Openstack.", + type=non_empty_string, + ) + run_parser.add_argument( + "-b", + "--base-image", dest="base", required=False, choices=tuple( @@ -141,59 +85,113 @@ def main(args: list[str] | None = None) -> None: (tag, name) for (tag, name) in LTS_IMAGE_VERSION_TAG_MAP.items() ) ), - default="jammy", - ) - build_parser.add_argument( - "-c", - "--cloud-name", - dest="cloud_name", - required=True, - help=( - "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " - "of the following order: current directory, ~/.config/openstack, /etc/openstack." - ), + default="noble", ) - build_parser.add_argument( - "-n", - "--num-revisions", - dest="num_revisions", + run_parser.add_argument( + "-k", + "--keep-revisions", + dest="keep_revisions", required=False, type=int, default=5, help="The maximum number of images to keep before deletion.", ) - build_parser.add_argument( - "-p", - "--callback-script-path", + run_parser.add_argument( + "-s", + "--callback-script", dest="callback_script_path", - required=True, + required=False, type=_existing_path, help=( "The callback script to trigger after image is built. The callback script is called" "with the first argument as the image ID." ), ) - build_parser.add_argument( - "-o", - "--output-image-name", - dest="image_name", - required=True, - help="The image name to upload to Openstack.", - ) - parsed = cast(ActionsNamespace, parser.parse_args(args)) - - if parsed.action == "install": - _install() + options = cast(ActionsNamespace, parser.parse_args(args)) + print(options) + if options.action == "init": + builder.initialize() return - if parsed.action == "get": - _get(cloud_name=parsed.cloud_name, image_name=parsed.image_name) + if options.action == "latest-build-id": + print( + store.get_latest_build_id( + cloud_name=options.cloud_name, image_name=options.image_name + ), + end=None, + ) return _build_and_upload( - base=parsed.base, - callback_script_path=parsed.callback_script_path, - cloud_name=parsed.cloud_name, - image_name=parsed.image_name, - num_revisions=parsed.num_revisions, + base=options.base, + cloud_name=options.cloud_name, + image_name=options.image_name, + keep_revisions=options.keep_revisions, + callback_script_path=options.callback_script_path, + ) + + +def _existing_path(value: str) -> Path: + """Check the path exists. + + Args: + value: The path string. + + Raises: + ValueError: If the path does not exist. + + Returns: + Path that exists. + """ + path = Path(value) + if not path.exists(): + raise ValueError(f"Given path {value} not found.") + return path + + +def non_empty_string(arg: str) -> str: + """Check that the argument is non-empty. + + Args: + arg: The argument to check. + + Raises: + ValueError: If the argument is empty. + + Returns: + Non-empty string. + """ + arg = str(arg) + if not arg: + raise ValueError("Must not be empty string") + return arg + + +def _build_and_upload( + base: str, + cloud_name: str, + image_name: str, + keep_revisions: int, + callback_script_path: Path | None = None, +) -> None: + """Build and upload image. + + Args: + base: Ubuntu image base. + cloud_name: The Openstack cloud to upload the image to. + image_name: The image name to upload as. + keep_revisions: Number of image revisions to keep before deletion. + callback_script_path: Path to bash script to call after image upload. + """ + arch = get_supported_arch() + base_image = BaseImage.from_str(base) + builder.build_image(arch=arch, base_image=base_image) + image_id = store.upload_image( + cloud_name=cloud_name, + image_name=image_name, + image_path=IMAGE_OUTPUT_PATH, + keep_revisions=keep_revisions, ) + if callback_script_path: + # The callback script is a user trusted script. + subprocess.check_call(["/bin/bash", str(callback_script_path), image_id]) # nosec: B603 diff --git a/src/github_runner_image_builder/config.py b/src/github_runner_image_builder/config.py index 70279c5..6527661 100644 --- a/src/github_runner_image_builder/config.py +++ b/src/github_runner_image_builder/config.py @@ -12,6 +12,10 @@ logger = logging.getLogger(__name__) +ACTION_INIT = "init" +ACTION_RUN = "run" +ACTION_LATEST_BUILD_ID = "latest-build-id" + # This is a class used for type hinting argparse. class ActionsNamespace(argparse.Namespace): # pylint: disable=too-few-public-methods @@ -24,15 +28,15 @@ class ActionsNamespace(argparse.Namespace): # pylint: disable=too-few-public-me cloud_name: The Openstack cloud to interact with. The CLI assumes clouds.yaml is written to the default path, i.e. current directory or ~/.config/openstack or /etc/openstack. image_name: The image name to upload as. - num_revisions: The maximum number of images to keep before deletion. + keep_revisions: The maximum number of images to keep before deletion. """ - action: Literal["install", "build", "get"] + action: Literal["init", "run", "latest-build-id"] base: Literal["22.04", "jammy", "24.04", "noble"] - callback_script_path: Path + callback_script_path: Path | None cloud_name: str image_name: str - num_revisions: int + keep_revisions: int class Arch(str, Enum): diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py index 8050ff7..428577a 100644 --- a/src/github_runner_image_builder/errors.py +++ b/src/github_runner_image_builder/errors.py @@ -29,8 +29,8 @@ class CleanBuildStateError(ImageBuilderBaseError): """Represents an error cleaning up build state.""" -class CloudImageDownloadError(ImageBuilderBaseError): - """Represents an error downloading cloud image.""" +class BaseImageDownloadError(ImageBuilderBaseError): + """Represents an error downloading base image.""" class ImageResizeError(ImageBuilderBaseError): @@ -57,8 +57,8 @@ class YQBuildError(ImageBuilderBaseError): """Represents an error while building yq binary from source.""" -class ExternalPackageInstallError(ImageBuilderBaseError): - """Represents an error installilng external packages.""" +class YarnInstallError(ImageBuilderBaseError): + """Represents an error installilng Yarn.""" class ImageCompressError(ImageBuilderBaseError): @@ -85,5 +85,5 @@ class GetImageError(OpenstackBaseError): """Represents an error when fetching images from Openstack.""" -class OpenstackConnectionError(OpenstackBaseError): +class OpenstackError(OpenstackBaseError): """Represents an error while communicating with Openstack.""" diff --git a/src/github_runner_image_builder/store.py b/src/github_runner_image_builder/store.py new file mode 100644 index 0000000..ac249a5 --- /dev/null +++ b/src/github_runner_image_builder/store.py @@ -0,0 +1,115 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for uploading images to shareable storage.""" + +import logging +from pathlib import Path +from typing import cast + +import openstack +import openstack.connection +import openstack.exceptions +from openstack.image.v2.image import Image + +from github_runner_image_builder.errors import GetImageError, OpenstackError, UploadImageError + +logger = logging.getLogger(__name__) + + +def upload_image(cloud_name: str, image_name: str, image_path: Path, keep_revisions: int) -> str: + """Upload image to openstack glance. + + Args: + cloud_name: The Openstack cloud to use from clouds.yaml. + image_name: The image name to upload as. + image_path: The path to image to upload. + keep_revisions: The number of revisions to keep for an image. + + Raises: + UploadImageError: If there was an error uploading the image to Openstack Glance. + + Returns: + The created image ID. + """ + with openstack.connect(cloud=cloud_name) as connection: + try: + image: Image = connection.create_image( + name=image_name, + filename=str(image_path), + allow_duplicates=True, + wait=True, + ) + _prune_old_images( + connection=connection, image_name=image_name, num_revisions=keep_revisions + ) + return image.id + except openstack.exceptions.OpenStackCloudException as exc: + raise UploadImageError from exc + + +def _prune_old_images( + connection: openstack.connection.Connection, image_name: str, num_revisions: int +) -> None: + """Remove old images outside of number of revisions to keep. + + Args: + connection: The connected openstack cloud instance. + image_name: The image name to search for. + num_revisions: The number of revisions to keep. + """ + images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) + images_to_prune = images[num_revisions:] + for image in images_to_prune: + try: + if not connection.delete_image(image.id, wait=True): + logger.error("Failed to delete old image, %s", image.id) + except openstack.exceptions.OpenStackCloudException as exc: + logger.error("Failed to prune old image, %s", exc) + continue + + +def _get_sorted_images_by_created_at( + connection: openstack.connection.Connection, image_name: str +) -> list[Image]: + """Fetch the images sorted by created_at date. + + Args: + connection: The connected openstack cloud instance. + image_name: The image name to search for. + + Raises: + OpenstackError: if there was an error fetching the images. + + Returns: + The images sorted by created_at date with latest first. + """ + try: + images = cast(list[Image], connection.search_images(image_name)) + except openstack.exceptions.OpenStackCloudException as exc: + raise OpenstackError from exc + + return sorted(images, key=lambda image: image.created_at, reverse=True) + + +def get_latest_build_id(cloud_name: str, image_name: str) -> str | None: + """Fetch the latest image id. + + Args: + cloud_name: The Openstack cloud to use from clouds.yaml. + image_name: The image name to search for. + + Raises: + GetImageError: If there was an error fetching image from Openstack. + + Returns: + The image ID if exists, None otherwise. + """ + with openstack.connect(cloud=cloud_name) as connection: + try: + images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) + except OpenstackError as exc: + raise GetImageError from exc + if not images: + return None + return images[0].id diff --git a/src/github_runner_image_builder/upload.py b/src/github_runner_image_builder/upload.py deleted file mode 100644 index bdd2b18..0000000 --- a/src/github_runner_image_builder/upload.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Module for uploading images to shareable storage.""" - -import dataclasses -import logging -from pathlib import Path -from typing import Any, cast - -import openstack -import openstack.connection -import openstack.exceptions -from openstack.image.v2.image import Image - -from github_runner_image_builder.config import Arch, BaseImage -from github_runner_image_builder.errors import ( - GetImageError, - OpenstackConnectionError, - UnauthorizedError, - UploadImageError, -) - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass -class UploadImageConfig: - """Configuration values for creating image. - - Attributes: - arch: The architecture the image was built for. - base: The ubuntu OS base the image was created with. - image_name: The image name to upload as. - num_revisions: The number of revisions to keep for an image. - src_path: The path to image to upload. - """ - - arch: Arch - base: BaseImage - image_name: str - num_revisions: int - src_path: Path - - -class OpenstackManager: - """Class to manage interactions with Openstack.""" - - def __init__(self, cloud_name: str): - """Initialize the openstack manager class. - - Args: - cloud_name: The Openstack cloud to use. - - Raises: - UnauthorizedError: If an invalid openstack credentials was given. - """ - try: - with openstack.connect(cloud=cloud_name) as conn: - conn.authorize() - # pylint thinks this isn't an exception, but does inherit from Exception class. - except openstack.exceptions.HttpException as exc: # pylint: disable=bad-exception-cause - raise UnauthorizedError("Unauthorized credentials.") from exc - - self.conn = openstack.connect(cloud_name) - - def __enter__(self) -> "OpenstackManager": - """Dunder method placeholder for context management. - - Returns: - Self with established connection. - """ - return self - - def __exit__(self, *_args: Any, **_kwargs: Any) -> None: - """Dunder method to close initialized connection to openstack.""" - self.conn.close() - - def _get_images_by_latest(self, image_name: str) -> list[Image]: - """Fetch the images sorted by latest. - - Args: - image_name: The image name to search for. - - Raises: - OpenstackConnectionError: if there was an error fetching the images. - - Returns: - The images sorted in latest first order. - """ - try: - images = cast(list[Image], self.conn.search_images(image_name)) - except openstack.exceptions.OpenStackCloudException as exc: - raise OpenstackConnectionError from exc - - return sorted(images, key=lambda image: image.created_at, reverse=True) - - def _prune_old_images(self, image_name: str, num_revisions: int) -> None: - """Remove old images outside of number of revisions to keep. - - Args: - image_name: The image name to search for. - num_revisions: The number of revisions to keep. - """ - images = self._get_images_by_latest(image_name=image_name) - images_to_prune = images[num_revisions:] - for image in images_to_prune: - try: - if not self.conn.delete_image(image.id, wait=True): - logger.error("Failed to delete old image, %s", image.id) - except openstack.exceptions.OpenStackCloudException as exc: - logger.error("Failed to prune old image, %s", exc) - continue - - def upload_image(self, config: UploadImageConfig) -> str: - """Upload image to openstack glance. - - Args: - config: Configuration values for creating image. - - Raises: - UploadImageError: If there was an error uploading the image to Openstack Glance. - - Returns: - The created image ID. - """ - try: - self._prune_old_images( - image_name=config.image_name, num_revisions=config.num_revisions - 1 - ) - image: Image = self.conn.create_image( - name=config.image_name, - filename=str(config.src_path), - allow_duplicates=True, - wait=True, - ) - return image.id - except openstack.exceptions.OpenStackCloudException as exc: - raise UploadImageError from exc - - def get_latest_image_id(self, image_name: str) -> str | None: - """Fetch the latest image id. - - Args: - image_name: The image name to search for. - - Raises: - GetImageError: If there was an error fetching image from Openstack. - - Returns: - The image ID if exists, None otherwise. - """ - try: - images = self._get_images_by_latest(image_name=image_name) - except OpenstackConnectionError as exc: - raise GetImageError from exc - if not images: - return None - return images[0].id diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 6a26c9f..8f357e1 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -71,6 +71,9 @@ def _create_metadata_tar_gz(image: str, tmp_path: Path) -> Path: Args: image: The ubuntu LTS image name. tmp_path: Temporary dir. + + Returns: + The path to created metadata.tar. """ # Create metadata.yaml template = Template( @@ -107,6 +110,9 @@ def _post_vm_img( image_data: Image qcow2 (.img) file contents in bytes. metadata: The metadata.tar.gz contents in bytes. public: Whether the image should be publicly available. + + Returns: + The created LXD Image instance. """ headers = {} if public: diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 852eab3..1ab2f42 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -17,24 +17,23 @@ from github_runner_image_builder.builder import ( Arch, BaseImage, + BaseImageDownloadError, BuilderSetupError, BuildImageError, ChrootBaseError, CleanBuildStateError, - CloudImageDownloadError, DependencyInstallError, - ExternalPackageInstallError, ImageCompressError, ImageMountError, ImageResizeError, NetworkBlockDeviceError, ResizePartitionError, - SupportedCloudImageArch, + SupportedBaseImageArch, SystemUserConfigurationError, UnattendedUpgradeDisableError, UnsupportedArchitectureError, + YarnInstallError, YQBuildError, - os, shutil, subprocess, ) @@ -48,7 +47,7 @@ def test__install_dependencies_package_not_found(monkeypatch: pytest.MonkeyPatch """ monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock( side_effect=[None, None, subprocess.CalledProcessError(1, [], "Package not found.")] ), @@ -60,10 +59,10 @@ def test__install_dependencies_package_not_found(monkeypatch: pytest.MonkeyPatch assert "Package not found" in str(exc.getrepr()) -def test__enable_nbd_fail(monkeypatch: pytest.MonkeyPatch): +def test__enable_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given subprocess run that raises CalledProcessError. - act: when _enable_nbd is called. + act: when _enable_network_block_device is called. assert: NetworkBlockDeviceError is raised. """ monkeypatch.setattr( @@ -73,7 +72,7 @@ def test__enable_nbd_fail(monkeypatch: pytest.MonkeyPatch): ) with pytest.raises(NetworkBlockDeviceError) as exc: - builder._enable_nbd() + builder._enable_network_block_device() assert "Module nbd not found" in str(exc.getrepr()) @@ -90,7 +89,7 @@ def test__enable_nbd_fail(monkeypatch: pytest.MonkeyPatch): ), pytest.param( builder, - "_enable_nbd", + "_enable_network_block_device", NetworkBlockDeviceError("Unable to enable nbd"), "Unable to enable nbd", id="Failed to enable nbd", @@ -111,11 +110,11 @@ def test_setup_builder_fail( """ mock_func = MagicMock(side_effect=exception) monkeypatch.setattr(builder, "_install_dependencies", MagicMock) - monkeypatch.setattr(builder, "_enable_nbd", MagicMock) + monkeypatch.setattr(builder, "_enable_network_block_device", MagicMock) monkeypatch.setattr(patch_obj, sub_func, mock_func) with pytest.raises(BuilderSetupError) as exc: - builder.setup_builder() + builder.initialize() assert expected_message in str(exc.getrepr()) @@ -138,7 +137,7 @@ def test__get_supported_runner_arch_unsupported_error(): pytest.param(Arch.X64, "amd64", id="AMD64"), ], ) -def test__get_supported_runner_arch(arch: Arch, expected: SupportedCloudImageArch): +def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch): """ arrange: given an architecture value that is supported. act: when _get_supported_runner_arch is called. @@ -172,14 +171,11 @@ def test__clean_build_state(monkeypatch: pytest.MonkeyPatch): act: when _clean_build_state is called. assert: the mocks are called. """ - mock_mount_dir = MagicMock() mock_subprocess_run = MagicMock() - monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", mock_mount_dir) monkeypatch.setattr(builder.subprocess, "run", mock_subprocess_run) builder._clean_build_state() - mock_mount_dir.mkdir.assert_called_once() mock_subprocess_run.assert_called() @@ -202,7 +198,7 @@ def test__clean_build_state(monkeypatch: pytest.MonkeyPatch): ), ], ) -def test__download_cloud_image_fail( +def test__download_base_image_fail( patch_obj: Any, sub_func: str, exception: Exception, @@ -210,8 +206,8 @@ def test__download_cloud_image_fail( monkeypatch: pytest.MonkeyPatch, ): """ - arrange: given monkeypatched sub functions of _download_cloud_image that raises exceptions. - act: when _download_cloud_image is called. + arrange: given monkeypatched sub functions of _download_base_image that raises exceptions. + act: when _download_base_image is called. assert: A CloudImageDownloadError is raised. """ mock_func = MagicMock(side_effect=exception) @@ -219,30 +215,30 @@ def test__download_cloud_image_fail( monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock) monkeypatch.setattr(patch_obj, sub_func, mock_func) - with pytest.raises(CloudImageDownloadError) as exc: - builder._download_cloud_image(arch=MagicMock(), base_image=MagicMock()) + with pytest.raises(BaseImageDownloadError) as exc: + builder._download_base_image(arch=MagicMock(), base_image=MagicMock()) assert expected_message in str(exc.getrepr()) -def test__download_cloud_image(monkeypatch: pytest.MonkeyPatch): +def test__download_base_image(monkeypatch: pytest.MonkeyPatch): """ - arrange: given monkeypatched sub functions of _download_cloud_image. - act: when _download_cloud_image is called. + arrange: given monkeypatched sub functions of _download_base_image. + act: when _download_base_image is called. assert: the downloaded path is returned. """ monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock(return_value="amd64")) monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock()) - assert builder._download_cloud_image(arch=Arch.X64, base_image=BaseImage.JAMMY) == Path( + assert builder._download_base_image(arch=Arch.X64, base_image=BaseImage.JAMMY) == Path( "jammy-server-cloudimg-amd64.img" ) -def test__resize_cloud_img_fail(monkeypatch: pytest.MonkeyPatch): +def test__resize_image_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched subprocess.run that raises an exception. - act: when _resize_cloud_img is called. + act: when _resize_image is called. assert: ImageResizeError is raised. """ mock_run = MagicMock( @@ -257,20 +253,20 @@ def test__resize_cloud_img_fail(monkeypatch: pytest.MonkeyPatch): ) with pytest.raises(ImageResizeError) as exc: - builder._resize_cloud_img(cloud_image_path=MagicMock()) + builder._resize_image(image_path=MagicMock()) assert "resize error" in str(exc.getrepr()) -def test__mount_nbd_partition(monkeypatch: pytest.MonkeyPatch): +def test__mount_network_block_device_partition(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched mock subprocess run. - act: when _mount_nbd_partition is called. + act: when _mount_network_block_device_partition is called. assert: subprocess run call is made. """ monkeypatch.setattr(subprocess, "run", (mock_run_call := MagicMock())) - builder._mount_nbd_partition() + builder._mount_network_block_device_partition() mock_run_call.assert_called_once() @@ -288,21 +284,24 @@ def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPat ) with pytest.raises(ImageMountError) as exc: - builder._mount_image_to_network_block_device(cloud_image_path=MagicMock()) + builder._mount_image_to_network_block_device(image_path=MagicMock()) assert "error mounting" in str(exc.getrepr()) def test__mount_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): """ - arrange: given a monkeypatched mock process run calls and _mount_nbd_partition call. + arrange: given a monkeypatched mock process run calls and \ + _mount_network_block_device_partition call. act: when _mount_image_to_network_block_device is called. assert: expected calls are made. """ monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) - monkeypatch.setattr(builder, "_mount_nbd_partition", (mount_mock := MagicMock())) + monkeypatch.setattr( + builder, "_mount_network_block_device_partition", (mount_mock := MagicMock()) + ) - builder._mount_image_to_network_block_device(cloud_image_path=MagicMock()) + builder._mount_image_to_network_block_device(image_path=MagicMock()) run_mock.assert_called_once() mount_mock.assert_called_once() @@ -392,20 +391,6 @@ def test__install_yq(monkeypatch: pytest.MonkeyPatch): copy_mock.assert_called() -def test__create_python_symlinks(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a monkeypatched os.symlink function. - act: when _create_python_symlinks is called. - assert: the symlink function is called. - """ - mock_symlink_call = MagicMock() - monkeypatch.setattr(os, "symlink", mock_symlink_call) - - builder._create_python_symlinks() - - mock_symlink_call.assert_called() - - def test__disable_unattended_upgrades_subprocess_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched subprocess run function that raises SubprocessError. @@ -450,10 +435,10 @@ def test__configure_system_users(monkeypatch: pytest.MonkeyPatch): assert "Failed to add group." in str(exc.getrepr()) -def test__install_external_packages_error(monkeypatch: pytest.MonkeyPatch): +def test__install_yarn_error(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched subprocess.run that raises an error. - act: when _install_external_packages is called. + act: when _install_yarn is called. assert: ExternalPackageInstallError is raised. """ # The test mocks use similar codes. @@ -468,21 +453,21 @@ def test__install_external_packages_error(monkeypatch: pytest.MonkeyPatch): ), ) - with pytest.raises(ExternalPackageInstallError) as exc: - builder._install_external_packages() + with pytest.raises(YarnInstallError) as exc: + builder._install_yarn() assert "Failed to clean npm cache." in str(exc.getrepr()) -def test__install_external_packages(monkeypatch: pytest.MonkeyPatch): +def test__install_yarn(monkeypatch: pytest.MonkeyPatch): """ - arrange: given a monkeypatched functions of _install_external_packages. - act: when _install_external_packages is called. + arrange: given a monkeypatched functions of _install_yarn. + act: when _install_yarn is called. assert: The function exists without raising an error. """ monkeypatch.setattr(subprocess, "run", MagicMock()) - assert builder._install_external_packages() is None + assert builder._install_yarn() is None def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): @@ -543,23 +528,23 @@ def test_build_image_error( act: when build_image is called. assert: BuildImageError is raised. """ + monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", MagicMock()) monkeypatch.setattr(builder, "_clean_build_state", MagicMock()) - monkeypatch.setattr(builder, "_download_cloud_image", MagicMock()) - monkeypatch.setattr(builder, "_resize_cloud_img", MagicMock()) + monkeypatch.setattr(builder, "_download_base_image", MagicMock()) + monkeypatch.setattr(builder, "_resize_image", MagicMock()) monkeypatch.setattr(builder, "_mount_image_to_network_block_device", MagicMock()) monkeypatch.setattr(builder, "_resize_mount_partitions", MagicMock()) monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) monkeypatch.setattr(builder, "_install_yq", MagicMock()) monkeypatch.setattr(builder, "ChrootContextManager", MagicMock()) monkeypatch.setattr(builder.subprocess, "run", MagicMock()) - monkeypatch.setattr(builder, "_create_python_symlinks", MagicMock()) monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) - monkeypatch.setattr(builder, "_install_external_packages", MagicMock()) + monkeypatch.setattr(builder, "_install_yarn", MagicMock()) monkeypatch.setattr(builder, "_compress_image", MagicMock()) monkeypatch.setattr(patch_obj, sub_func, mock) with pytest.raises(BuildImageError) as exc: - builder.build_image(config=MagicMock()) + builder.build_image(arch=MagicMock(), base_image=MagicMock()) assert expected_message in str(exc.getrepr()) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 11cbac5..8ea7aa7 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -24,15 +24,15 @@ def callback_path_fixture(tmp_path: Path): return test_path -@pytest.fixture(scope="function", name="build_image_inputs") -def build_image_inputs_fixture(callback_path: Path): - """Valid CLI inputs.""" +@pytest.fixture(scope="function", name="run_inputs") +def run_inputs_fixture(callback_path: Path): + """Valid CLI run mode inputs.""" return { - "-i": "jammy", - "-c": "test-cloud-name", - "-n": "5", - "-p": str(callback_path), - "-o": "test-image", + "": "test-cloud-name", + " ": "test-image-name", + "--base-image": "noble", + "--keep-revisions": "5", + "--callback-script": str(callback_path), } @@ -49,57 +49,32 @@ def test__existing_path(tmp_path: Path): assert f"Given path {not_exists_path} not found." in str(exc.getrepr()) -def test__install(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a monkeypatched builder.setup_builder function. - act: when _install is called. - assert: the mock function is called. - """ - monkeypatch.setattr(cli.builder, "setup_builder", (setup_mock := MagicMock())) - - cli._install() - - setup_mock.assert_called_once() - - -def test__get(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture): - """ - arrange: given a monkeypatched OpenstackManager.get_latest_image_id. - act: when _get is called. - assert: the image id is captured in stdout. - """ - openstack_manager_mock = MagicMock() - openstack_manager_mock.__enter__.return_value = (openstack_mock := MagicMock()) - openstack_mock.get_latest_image_id.return_value = "testing_id" - monkeypatch.setattr(cli, "OpenstackManager", MagicMock(return_value=openstack_manager_mock)) - - cli._get(cloud_name=MagicMock(), image_name=MagicMock()) - - res = capsys.readouterr() - assert "testing_id" in res.out - - -def test__build_and_upload(monkeypatch: pytest.MonkeyPatch, callback_path: Path): +@pytest.mark.parametrize( + "callback_script", + [ + pytest.param(None, id="No callback script"), + pytest.param(Path("tmp_path"), id="Callback script"), + ], +) +def test__build_and_upload(monkeypatch: pytest.MonkeyPatch, callback_script: Path | None): """ arrange: given a monkeypatched builder.setup_builder function. act: when _build is called. assert: the mock function is called. """ monkeypatch.setattr(cli.builder, "build_image", (builder_mock := MagicMock())) - monkeypatch.setattr( - cli, "OpenstackManager", MagicMock(return_value=(openstack_manager := MagicMock())) - ) + monkeypatch.setattr(cli.store, "upload_image", MagicMock(return_value="test-image-id")) + monkeypatch.setattr(cli.subprocess, "check_call", MagicMock()) cli._build_and_upload( base="jammy", - callback_script_path=callback_path, cloud_name=MagicMock(), image_name=MagicMock(), - num_revisions=MagicMock(), + keep_revisions=MagicMock(), + callback_script_path=callback_script, ) builder_mock.assert_called_once() - openstack_manager.__enter__.return_value.upload_image.assert_called_once() @pytest.mark.parametrize( @@ -115,48 +90,48 @@ def test_main_invalid_choice(monkeypatch: pytest.MonkeyPatch, choice: str): act: when main is called. assert: SystemExit is raised and mocked builder functions are not called. """ - monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) - monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) + monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) with pytest.raises(SystemExit): main([choice]) - install_mock.assert_not_called() + initialize_mock.assert_not_called() get_mock.assert_not_called() build_mock.assert_not_called() -def test_main_install(monkeypatch: pytest.MonkeyPatch): +def test_main_init(monkeypatch: pytest.MonkeyPatch): """ - arrange: given install argument and mocked builder functions. + arrange: given init argument and mocked builder functions. act: when main is called. - assert: install builder mock function is called. + assert: initialize builder mock function is called. """ - monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) - monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) + monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - main(["install"]) + main(["init"]) - install_mock.assert_called() + initialize_mock.assert_called() get_mock.assert_not_called() build_mock.assert_not_called() -def test_main_get(monkeypatch: pytest.MonkeyPatch): +def test_main_latest_build_id(monkeypatch: pytest.MonkeyPatch): """ - arrange: given install argument and mocked builder functions. + arrange: given latest-build-id argument and mocked builder functions. act: when main is called. assert: get mock function is called. """ - monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) - monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) + monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - main(["get", "-c", "test-cloud", "-o", "test-output-image-name"]) + main(["latest-build-id", "test-cloud", "test-image"]) - install_mock.assert_not_called() + initialize_mock.assert_not_called() get_mock.assert_called() build_mock.assert_not_called() @@ -164,34 +139,37 @@ def test_main_get(monkeypatch: pytest.MonkeyPatch): @pytest.mark.parametrize( "invalid_patch", [ - pytest.param({"-i": ""}, id="no base-image"), - pytest.param({"-i": "test"}, id="invalid base-image"), + pytest.param({"--base-image": ""}, id="no base-image"), + pytest.param({"--base-image": "test"}, id="invalid base-image"), + pytest.param({"": ""}, id="empty cloud name positional argument"), + pytest.param({" ": ""}, id="empty image name positional argument"), ], ) -def test_main_invalid_build_inputs( +def test_main_invalid_run_inputs( monkeypatch: pytest.MonkeyPatch, - build_image_inputs: dict[str, str], + run_inputs: dict[str, str], invalid_patch: dict[str, str], ): """ - arrange: given invalid build arguments and mocked builder functions. + arrange: given invalid run arguments and mocked builder functions. act: when main is called. assert: SystemExit is raised. """ - monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) - monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) + monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - build_image_inputs.update(invalid_patch) + run_inputs.update(invalid_patch) inputs = list( + # if flag does not exist, append it as a positional argument. itertools.chain.from_iterable( - (flag, value) for (flag, value) in build_image_inputs.items() + (flag, value) if flag.strip() else (value,) for (flag, value) in run_inputs.items() ) ) with pytest.raises(SystemExit): - main(["build", *inputs]) + main(["run", *inputs]) - install_mock.assert_not_called() + initialize_mock.assert_not_called() get_mock.assert_not_called() build_mock.assert_not_called() @@ -205,26 +183,25 @@ def test_main_invalid_build_inputs( pytest.param("24.04", id="noble tag"), ], ) -def test_main_base_image( - monkeypatch: pytest.MonkeyPatch, image: str, build_image_inputs: dict[str, str] -): +def test_main_run(monkeypatch: pytest.MonkeyPatch, image: str, run_inputs: dict[str, str]): """ - arrange: given invalid base_image argument and mocked builder functions. + arrange: given invalid run argument and mocked builder functions. act: when main is called. - assert: build image is called. + assert: run is called. """ - monkeypatch.setattr(cli, "_install", (install_mock := MagicMock())) - monkeypatch.setattr(cli, "_get", (get_mock := MagicMock())) + monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) + monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - build_image_inputs.update({"-i": image}) + run_inputs.update({"--base-image": image}) inputs = list( + # if flag does not exist, append it as a positional argument. itertools.chain.from_iterable( - (flag, value) for (flag, value) in build_image_inputs.items() + (flag, value) if flag.strip() else (value,) for (flag, value) in run_inputs.items() ) ) - main(["build", *inputs]) + main(["run", *inputs]) - install_mock.assert_not_called() + initialize_mock.assert_not_called() get_mock.assert_not_called() build_mock.assert_called() diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py new file mode 100644 index 0000000..1c68ba9 --- /dev/null +++ b/tests/unit/test_store.py @@ -0,0 +1,210 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for builder module.""" + +# Need access to protected functions for testing +# pylint:disable=protected-access + +from unittest.mock import MagicMock + +import pytest +from openstack.connection import Connection + +from github_runner_image_builder import store +from github_runner_image_builder.store import ( + GetImageError, + Image, + OpenstackError, + UploadImageError, + openstack, +) +from tests.unit.factories import MockOpenstackImageFactory + + +# Fixture docstrings do not need argument or return values. +@pytest.fixture(name="mock_connection") +def mock_connection_fixture(monkeypatch: pytest.MonkeyPatch) -> Connection: + """Mock the openstack connection instance.""" # noqa: DCO020 + connection_mock = MagicMock() + connection_context_mock = MagicMock(spec=Connection) + connection_mock.__enter__.return_value = connection_context_mock + monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connection_mock)) + return connection_context_mock # noqa: DCO030 + + +def test__get_sorted_images_by_created_at_error(mock_connection: MagicMock): + """ + arrange: given a mocked openstack connection that returns images in non-sorted order. + act: when _get_sorted_images_by_created_at is called. + assert: the images are returned in sorted order by creation date. + """ + mock_connection.search_images.side_effect = openstack.exceptions.OpenStackCloudException( + "Network error" + ) + + with pytest.raises(OpenstackError) as err: + store._get_sorted_images_by_created_at(connection=mock_connection, image_name=MagicMock) + + assert "Network error" in str(err.getrepr()) + + +def test__get_sorted_images_by_created_at(mock_connection: MagicMock): + """ + arrange: given a mocked openstack connection that returns images in non-sorted order. + act: when _get_sorted_images_by_created_at is called. + assert: the images are returned in sorted order by creation date. + """ + mock_connection.search_images.return_value = [ + (first := MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z")), + (third := MockOpenstackImageFactory(id="3", created_at="2024-03-03T00:00:00Z")), + (second := MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z")), + ] + + assert store._get_sorted_images_by_created_at( + connection=mock_connection, image_name=MagicMock + ) == [third, second, first] + + +def test__prune_old_images_error(caplog: pytest.LogCaptureFixture, mock_connection: MagicMock): + """ + arrange: given a mocked delete function that raises an exception. + act: when _prune_old_images is called. + assert: failure to delete is logged. + """ + mock_connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + mock_connection.delete_image.side_effect = openstack.exceptions.OpenStackCloudException( + "Delete error" + ) + + store._prune_old_images(connection=mock_connection, image_name=MagicMock(), num_revisions=0) + + assert all("Failed to prune old image" in log for log in caplog.messages) + + +def test__prune_old_images_fail(caplog: pytest.LogCaptureFixture, mock_connection: MagicMock): + """ + arrange: given a mocked delete function that returns false. + act: when _prune_old_images is called. + assert: failure to delete is logged. + """ + mock_connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + mock_connection.delete_image.return_value = False + + store._prune_old_images(connection=mock_connection, image_name=MagicMock(), num_revisions=0) + + assert all("Failed to delete old image" in log for log in caplog.messages) + + +def test__prune_old_images(mock_connection: MagicMock): + """ + arrange: given a mocked delete function that returns true. + act: when _prune_old_images is called. + assert: delete mock is called. + """ + mock_connection.search_images.return_value = [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ] + mock_connection.delete_image.return_value = True + + store._prune_old_images(connection=mock_connection, image_name=MagicMock(), num_revisions=0) + + assert mock_connection.delete_image.call_count == 2 + + +def test_upload_image_error(mock_connection: MagicMock): + """ + arrange: given a mocked openstack create_image function that raises an exception. + act: when upload_image is called. + assert: UploadImageError is raised. + """ + mock_connection.create_image.side_effect = openstack.exceptions.OpenStackCloudException( + "Resource capacity exceeded." + ) + + with pytest.raises(UploadImageError) as exc: + store.upload_image( + cloud_name=MagicMock(), + image_name=MagicMock(), + image_path=MagicMock(), + keep_revisions=MagicMock(), + ) + + assert "Resource capacity exceeded." in str(exc.getrepr()) + + +def test_upload_image(mock_connection: MagicMock): + """ + arrange: given a mocked openstack create_image function that raises an exception. + act: when upload_image is called. + assert: UploadImageError is raised. + """ + mock_connection.create_image.return_value = MockOpenstackImageFactory(id="1") + + assert ( + store.upload_image( + cloud_name=MagicMock(), + image_name=MagicMock(), + image_path=MagicMock(), + keep_revisions=MagicMock(), + ) + == "1" + ) + + +@pytest.mark.usefixtures("mock_connection") +def test_get_latest_image_id_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a mocked _get_images_by_latest function that raises an exception. + act: when get_latest_image_id is called. + assert: GetImageError is raised. + """ + monkeypatch.setattr( + store, + "_get_sorted_images_by_created_at", + MagicMock(side_effect=OpenstackError("Unauthorized")), + ) + + with pytest.raises(GetImageError) as exc: + store.get_latest_build_id(cloud_name=MagicMock(), image_name=MagicMock()) + + assert "Unauthorized" in str(exc.getrepr()) + + +@pytest.mark.usefixtures("mock_connection") +@pytest.mark.parametrize( + "images, expected_id", + [ + pytest.param([], None, id="No images"), + pytest.param( + [ + MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), + MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), + ], + "1", + id="Multiple images", + ), + ], +) +def test_get_latest_image_id( + images: list[Image], expected_id: str | None, monkeypatch: pytest.MonkeyPatch +): + """ + arrange: given a mocked _get_images_by_latest function that returns openstack images. + act: when get_latest_image_id is called. + assert: GetImageError is raised. + """ + monkeypatch.setattr( + store, + "_get_sorted_images_by_created_at", + MagicMock(return_value=images), + ) + + assert store.get_latest_build_id(cloud_name=MagicMock(), image_name=MagicMock()) == expected_id diff --git a/tests/unit/test_upload.py b/tests/unit/test_upload.py deleted file mode 100644 index 5e48663..0000000 --- a/tests/unit/test_upload.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Unit tests for builder module.""" - -# Need access to protected functions for testing -# pylint:disable=protected-access - -from unittest.mock import MagicMock - -import pytest - -from github_runner_image_builder.upload import ( - GetImageError, - Image, - OpenstackConnectionError, - OpenstackManager, - UnauthorizedError, - UploadImageError, - openstack, -) -from tests.unit.factories import MockOpenstackImageFactory - - -@pytest.fixture(name="connection") -def mocked_openstack_connection_fixture(): - """Fixture for Openstack connection context manager mock instance.""" - connection_mock = MagicMock() - return connection_mock - - -@pytest.fixture(name="manager") -def openstack_manager_mock_fixture(monkeypatch: pytest.MonkeyPatch, connection: MagicMock): - """Fixture for OpenstackManager.""" - monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connection)) - return OpenstackManager(cloud_name="test") - - -def test_openstack_manager_context( - monkeypatch: pytest.MonkeyPatch, connection: MagicMock, manager: OpenstackManager -): - """ - arrange: given a monkeypatched openstack connection. - act: when openstck manager context is entered and exited. - assert: connection is closed. - """ - monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connection)) - - with manager: - pass - - connection.close.assert_called_once() - - -def test___init__error(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a monkeypatched openstack authorize function that raises an exception. - act: when OpenstackManager is initialized. - assert: UnauthorizedError is raised. - """ - connect_mock = MagicMock() - connect_mock.__enter__.return_value = (connection_mock := MagicMock()) - connection_mock.authorize.side_effect = openstack.exceptions.HttpException - monkeypatch.setattr(openstack, "connect", MagicMock(return_value=connect_mock)) - - with pytest.raises(UnauthorizedError) as exc: - OpenstackManager(cloud_name="tests") - - assert "Unauthorized credentials." in str(exc.getrepr()) - - -def test__get_images_by_latest_error(connection: MagicMock, manager: OpenstackManager): - """ - arrange: given a mocked openstack connection that returns images in non-sorted order. - act: when _get_images_by_latest is called. - assert: the images are returned in sorted order by creation date. - """ - connection.search_images.side_effect = openstack.exceptions.OpenStackCloudException( - "Network error" - ) - - with pytest.raises(OpenstackConnectionError) as err: - manager._get_images_by_latest(image_name=MagicMock) - - assert "Network error" in str(err.getrepr()) - - -def test__get_images_by_latest(connection: MagicMock, manager: OpenstackManager): - """ - arrange: given a mocked openstack connection that returns images in non-sorted order. - act: when _get_images_by_latest is called. - assert: the images are returned in sorted order by creation date. - """ - connection.search_images.return_value = [ - (first := MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z")), - (third := MockOpenstackImageFactory(id="3", created_at="2024-03-03T00:00:00Z")), - (second := MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z")), - ] - - assert manager._get_images_by_latest(image_name=MagicMock) == [third, second, first] - - -def test__prune_old_images_error( - caplog: pytest.LogCaptureFixture, - connection: MagicMock, - manager: OpenstackManager, -): - """ - arrange: given a mocked delete function that raises an exception. - act: when _prune_old_images is called. - assert: failure to delete is logged. - """ - connection.search_images.return_value = [ - MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), - MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), - ] - connection.delete_image.side_effect = openstack.exceptions.OpenStackCloudException( - "Delete error" - ) - - manager._prune_old_images(image_name=MagicMock(), num_revisions=0) - - assert all("Failed to prune old image" in log for log in caplog.messages) - - -def test__prune_old_images_fail( - caplog: pytest.LogCaptureFixture, connection: MagicMock, manager: OpenstackManager -): - """ - arrange: given a mocked delete function that returns false. - act: when _prune_old_images is called. - assert: failure to delete is logged. - """ - connection.search_images.return_value = [ - MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), - MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), - ] - connection.delete_image.return_value = False - - manager._prune_old_images(image_name=MagicMock(), num_revisions=0) - - assert all("Failed to delete old image" in log for log in caplog.messages) - - -def test__prune_old_images(connection: MagicMock, manager: OpenstackManager): - """ - arrange: given a mocked delete function that returns true. - act: when _prune_old_images is called. - assert: delete mock is called. - """ - connection.search_images.return_value = [ - MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), - MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), - ] - connection.delete_image.return_value = True - - manager._prune_old_images(image_name=MagicMock(), num_revisions=0) - - assert connection.delete_image.call_count == 2 - - -def test_upload_image_error(connection: MagicMock, manager: OpenstackManager): - """ - arrange: given a mocked openstack create_image function that raises an exception. - act: when upload_image is called. - assert: UploadImageError is raised. - """ - connection.create_image.side_effect = openstack.exceptions.OpenStackCloudException( - "Resource capacity exceeded." - ) - - with pytest.raises(UploadImageError) as exc: - manager.upload_image(config=MagicMock()) - - assert "Resource capacity exceeded." in str(exc.getrepr()) - - -def test_upload_image(connection: MagicMock, manager: OpenstackManager): - """ - arrange: given a mocked openstack create_image function that raises an exception. - act: when upload_image is called. - assert: UploadImageError is raised. - """ - connection.create_image.return_value = MockOpenstackImageFactory(id="1") - - assert manager.upload_image(config=MagicMock()) == "1" - - -def test_get_latest_image_id_error(manager: OpenstackManager): - """ - arrange: given a mocked _get_images_by_latest function that raises an exception. - act: when get_latest_image_id is called. - assert: GetImageError is raised. - """ - manager._get_images_by_latest = MagicMock(side_effect=OpenstackConnectionError("Unauthorized")) - - with pytest.raises(GetImageError) as exc: - manager.get_latest_image_id(image_name=MagicMock()) - - assert "Unauthorized" in str(exc.getrepr()) - - -@pytest.mark.parametrize( - "images, expected_id", - [ - pytest.param([], None, id="No images"), - pytest.param( - [ - MockOpenstackImageFactory(id="1", created_at="2024-01-01T00:00:00Z"), - MockOpenstackImageFactory(id="2", created_at="2024-02-02T00:00:00Z"), - ], - "1", - id="Multiple images", - ), - ], -) -def test_get_latest_image_id( - manager: OpenstackManager, images: list[Image], expected_id: str | None -): - """ - arrange: given a mocked _get_images_by_latest function that returns openstack images. - act: when get_latest_image_id is called. - assert: GetImageError is raised. - """ - manager._get_images_by_latest = MagicMock(return_value=images) - - assert manager.get_latest_image_id(image_name=MagicMock()) == expected_id From 09b9f11badbe29c6229b4b1b81ef45ccbddf4d1e Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:28:58 +0000 Subject: [PATCH 06/63] integration test --- src/github_runner_image_builder/cli.py | 1 - tests/integration/conftest.py | 16 +++++++--------- tests/integration/test_image.py | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index 536a090..9a6c21d 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -108,7 +108,6 @@ def main(args: list[str] | None = None) -> None: ), ) options = cast(ActionsNamespace, parser.parse_args(args)) - print(options) if options.action == "init": builder.initialize() return diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 82f4c6f..0c3e6ab 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -133,20 +133,18 @@ def cli_run_fixture( openstack_image_name: str, ): """A CLI run.""" - main(["install"]) + main(["init"]) main( [ - "build", - "-i", - image, - "-c", + "run", cloud_name, - "-n", + openstack_image_name, + "--base-image", + image, + "--keep-revisions", "2", - "-p", + "--callback-script", str(callback_script), - "-o", - openstack_image_name, ] ) diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index 7096cd6..ee10ef6 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -130,7 +130,7 @@ async def test_get_image( act: when get image id is run. assert: the latest image matches the stdout output. """ - main(["get", "-c", cloud_name, "-o", openstack_image_name]) + main(["latest-build-id", cloud_name, openstack_image_name]) image_id = openstack_connection.get_image_id(openstack_image_name) res = capsys.readouterr() From 1121b3eb5ac510766c4ff3e9ce260ea5d3fca1a9 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:30:13 +0000 Subject: [PATCH 07/63] ci name for different archs --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8430323..8e332a5 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -9,7 +9,7 @@ concurrency: jobs: integration-tests-arm: - name: Integration test + name: Integration test (ARM64) runs-on: [self-hosted, ARM64] strategy: matrix: @@ -28,7 +28,7 @@ jobs: run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} integration-tests-amd: - name: Integration test + name: Integration test (X64) runs-on: [self-hosted, X64] strategy: matrix: From eaad3173564aee9ea9cf40d34143976e0194de3f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:34:17 +0000 Subject: [PATCH 08/63] install tox via pipx --- .github/workflows/integration_test.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8e332a5..3bd77fb 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -21,9 +21,13 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - # need to run in sudo mode due to chroot - name: Install tox - run: sudo python -m pip install tox-gh + run: | + sudo apt-get update + sudo apt-get install pipx -y + pipx ensurepath + pipx install tox + # need to run in sudo mode due to chroot - name: Run integration tests run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} @@ -40,8 +44,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - # need to run in sudo mode due to chroot - name: Install tox - run: sudo python -m pip install tox-gh + run: | + sudo apt-get update + sudo apt-get install pipx -y + pipx ensurepath + pipx install tox + # need to run in sudo mode due to chroot - name: Run integration tests run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} From 3c118bc18e7e18546bb8aeca22734491c5cf9fd1 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:36:34 +0000 Subject: [PATCH 09/63] fix mount step --- src/github_runner_image_builder/builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 966b600..3c0d653 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -160,10 +160,10 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: base_image_path = _download_base_image(arch=arch, base_image=base_image) logger.info("Resizing base image.") _resize_image(image_path=base_image_path) - logger.info("Replacing resolv.conf.") - _replace_mounted_resolv_conf() logger.info("Mounting network block device.") _mount_image_to_network_block_device(image_path=base_image_path) + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() logger.info("Resizing partitions.") _resize_mount_partitions() logger.info("Installing YQ from source.") From 7ad2689dc71fce8db50acfe48a384ea10de6c49a Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:49:13 +0000 Subject: [PATCH 10/63] add python version --- .github/workflows/integration_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 3bd77fb..4e9fa67 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -14,6 +14,7 @@ jobs: strategy: matrix: image: [jammy, noble] + py: "3.10" steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 @@ -37,6 +38,7 @@ jobs: strategy: matrix: image: [jammy, noble] + py: "3.10" steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 From fc38390a7bda8b317eaafe553a991d35d4794d03 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:50:09 +0000 Subject: [PATCH 11/63] update setup python actions version --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 4e9fa67..0357ac7 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 - name: Setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} - name: Install tox @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 - name: Setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} - name: Install tox From f74405cabbae765300b9a2dfa2ce6fbb6602fb69 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:53:13 +0000 Subject: [PATCH 12/63] matricize py --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 0357ac7..0078f9c 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -14,7 +14,7 @@ jobs: strategy: matrix: image: [jammy, noble] - py: "3.10" + py: ["3.10"] steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 @@ -38,7 +38,7 @@ jobs: strategy: matrix: image: [jammy, noble] - py: "3.10" + py: ["3.10"] steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 From 5b8314791e00986fc21cdf1ac847c3272acdffb3 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 18:59:52 +0000 Subject: [PATCH 13/63] use avail versions for arm64 py --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 0078f9c..a42c244 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -14,7 +14,7 @@ jobs: strategy: matrix: image: [jammy, noble] - py: ["3.10"] + py: ["3.10.11"] # The only 3.10 version w/ ARM64 support steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 From db5dc9dc9f7aef1cd61ce94e1f66aefa00467922 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:00:57 +0000 Subject: [PATCH 14/63] add comment for avail arm64 py versions --- .github/workflows/integration_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index a42c244..658592b 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -14,7 +14,9 @@ jobs: strategy: matrix: image: [jammy, noble] - py: ["3.10.11"] # The only 3.10 version w/ ARM64 support + # 2024.06.01 The only 3.10 version w/ ARM64 support is 3.10.11 + # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json + py: ["3.10.11"] steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 From ec55cce5c8bc030b6e6cc15498279a0732860e88 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:09:36 +0000 Subject: [PATCH 15/63] run on jammy builders --- .github/workflows/integration_test.yaml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 658592b..8d6a983 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,20 +10,13 @@ concurrency: jobs: integration-tests-arm: name: Integration test (ARM64) - runs-on: [self-hosted, ARM64] + runs-on: [self-hosted, ARM64, jammy] strategy: matrix: image: [jammy, noble] - # 2024.06.01 The only 3.10 version w/ ARM64 support is 3.10.11 - # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json - py: ["3.10.11"] steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.py }} - name: Install tox run: | sudo apt-get update @@ -36,7 +29,7 @@ jobs: integration-tests-amd: name: Integration test (X64) - runs-on: [self-hosted, X64] + runs-on: [self-hosted, X64, jammy] strategy: matrix: image: [jammy, noble] @@ -44,10 +37,6 @@ jobs: steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.py }} - name: Install tox run: | sudo apt-get update From 2630613243595af4aaddce177e2bbcac46fe5604 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:10:27 +0000 Subject: [PATCH 16/63] remove py matrix --- .github/workflows/integration_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 8d6a983..1a690dd 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -33,7 +33,6 @@ jobs: strategy: matrix: image: [jammy, noble] - py: ["3.10"] steps: - uses: actions/checkout@v3 - uses: canonical/setup-lxd@v0.1.1 From 7a37ea40c513ce74fd40bef488ad97a97a4a6546 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:20:23 +0000 Subject: [PATCH 17/63] clean build state before compress --- src/github_runner_image_builder/builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 3c0d653..9c6623d 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -199,6 +199,8 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: _install_yarn() except ChrootBaseError as exc: raise BuildImageError from exc + finally: + _clean_build_state() try: logger.info("Compressing image.") From 5859ad2aadd1230819c4c4a17ae90a1ea8991b26 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:34:44 +0000 Subject: [PATCH 18/63] use private endpoint runners (openstack integration test) --- tests/integration/test_image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index ee10ef6..816d31b 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -134,4 +134,6 @@ async def test_get_image( image_id = openstack_connection.get_image_id(openstack_image_name) res = capsys.readouterr() - assert res.out == image_id, f"Openstack image not matching, {res.out} {res.err}, {image_id}" + assert ( + res.out.strip() == image_id + ), f"Openstack image not matching, {res.out} {res.err}, {image_id}" From 9d7bd156c6ff39f2374d85aa31a94a85712755b8 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 31 May 2024 19:50:52 +0000 Subject: [PATCH 19/63] run on prv endpoint --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 1a690dd..94e9a2a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -10,7 +10,7 @@ concurrency: jobs: integration-tests-arm: name: Integration test (ARM64) - runs-on: [self-hosted, ARM64, jammy] + runs-on: [self-hosted, ARM64, jammy, stg-private-endpoint] strategy: matrix: image: [jammy, noble] @@ -29,7 +29,7 @@ jobs: integration-tests-amd: name: Integration test (X64) - runs-on: [self-hosted, X64, jammy] + runs-on: [self-hosted, X64, jammy, stg-private-endpoint] strategy: matrix: image: [jammy, noble] From 66cf0d9e6a00e690ff7104f49d1fceca38d25c2b Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 3 Jun 2024 01:34:11 +0000 Subject: [PATCH 20/63] capture log outputs --- src/github_runner_image_builder/builder.py | 197 ++++++++++++++------- src/github_runner_image_builder/chroot.py | 6 +- src/github_runner_image_builder/cli.py | 2 +- tests/integration/test_image.py | 4 +- 4 files changed, 138 insertions(+), 71 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 9c6623d..9603b60 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -100,7 +100,9 @@ def initialize() -> None: BuilderSetupError: If there was an error setting up the host device for building images. """ try: + logger.info("Installing dependencies.") _install_dependencies() + logger.info("Enabling network block device.") _enable_network_block_device() except ImageBuilderBaseError as exc: raise BuilderSetupError from exc @@ -127,6 +129,12 @@ def _install_dependencies() -> None: timeout=30 * 60, ) # nosec: B603 except subprocess.CalledProcessError as exc: + logger.exception( + "Error installing dependencies, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise DependencyInstallError from exc @@ -137,8 +145,14 @@ def _enable_network_block_device() -> None: NetworkBlockDeviceError: If there was an error enable nbd kernel. """ try: - subprocess.run(["/usr/sbin/modprobe", "nbd"], check=True, timeout=10) # nosec: B603 + subprocess.check_output(["/usr/sbin/modprobe", "nbd"], timeout=10) # nosec: B603 except subprocess.CalledProcessError as exc: + logger.exception( + "Error enabling network block device, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise NetworkBlockDeviceError from exc @@ -178,9 +192,10 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: # chroot env, hence use subprocess run. subprocess.run( ["/usr/bin/apt-get", "update", "-y"], - check=True, timeout=60 * 10, env={"DEBIAN_FRONTEND": "noninteractive"}, + check=True, + capture_output=True, ) # nosec: B603 subprocess.run( # nosec: B603 [ @@ -190,9 +205,10 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: "--no-install-recommends", *IMAGE_DEFAULT_APT_PACKAGES, ], - check=True, timeout=60 * 20, env={"DEBIAN_FRONTEND": "noninteractive"}, + check=True, + capture_output=True, ) _disable_unattended_upgrades() _configure_system_users() @@ -219,34 +235,51 @@ def _clean_build_state() -> None: # output of subprocess runs. try: subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], + timeout=30, + check=False, + capture_output=True, ) # nosec: B603 subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], timeout=30, check=False + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], + timeout=30, + check=False, + capture_output=True, ) # nosec: B603 subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], timeout=30, check=False + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], + timeout=30, + check=False, + capture_output=True, ) # nosec: B603 subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False, capture_output=True ) # nosec: B603 subprocess.run( - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], + timeout=30, + check=False, + capture_output=True, ) # nosec: B603 subprocess.run( # nosec: B603 - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + timeout=30, + check=False, + capture_output=True, ) subprocess.run( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False, + capture_output=True, ) subprocess.run( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False, + capture_output=True, ) - except subprocess.SubprocessError as exc: + except subprocess.CalledProcessError as exc: raise CleanBuildStateError from exc @@ -320,12 +353,17 @@ def _resize_image(image_path: Path) -> None: ImageResizeError: If there was an error resizing the image. """ try: - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-img", "resize", str(image_path), RESIZE_AMOUNT], - check=True, timeout=60, ) except subprocess.CalledProcessError as exc: + logger.exception( + "Error resizing image, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise ImageResizeError from exc @@ -338,7 +376,7 @@ def _replace_mounted_resolv_conf() -> None: @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _mount_network_block_device_partition() -> None: """Mount the network block device partition.""" - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 [ "/usr/bin/mount", "-o", @@ -346,7 +384,6 @@ def _mount_network_block_device_partition() -> None: str(NETWORK_BLOCK_DEVICE_PARTITION_PATH), str(IMAGE_MOUNT_DIR), ], - check=True, timeout=60, ) @@ -361,13 +398,18 @@ def _mount_image_to_network_block_device(image_path: Path) -> None: ImageMountError: If there was an error mounting the image to network block device. """ try: - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-nbd", f"--connect={NETWORK_BLOCK_DEVICE_PATH}", str(image_path)], - check=True, timeout=60, ) _mount_network_block_device_partition() except subprocess.CalledProcessError as exc: + logger.exception( + "Error mounting image to network block device, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise ImageMountError from exc @@ -378,15 +420,20 @@ def _resize_mount_partitions() -> None: ResizePartitionError: If there was an error resizing network block device partitions. """ try: - subprocess.run( # nosec: B603 - ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], check=True, timeout=10 * 60 + subprocess.check_output( # nosec: B603 + ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], timeout=10 * 60 ) - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/sbin/resize2fs", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], - check=True, timeout=10 * 60, ) except subprocess.CalledProcessError as exc: + logger.exception( + "Error resizing mount partitions, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise ResizePartitionError from exc @@ -398,24 +445,27 @@ def _install_yq() -> None: """ try: if not YQ_REPOSITORY_PATH.exists(): - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/bin/git", "clone", str(YQ_REPOSITORY_URL), str(YQ_REPOSITORY_PATH)], - check=True, timeout=60 * 10, ) else: - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/bin/git", "-C", str(YQ_REPOSITORY_PATH), "pull"], - check=True, timeout=60 * 10, ) - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/snap/bin/go", "build", "-C", str(YQ_REPOSITORY_PATH), "-o", str(HOST_YQ_BIN_PATH)], - check=True, timeout=20 * 60, ) shutil.copy(HOST_YQ_BIN_PATH, MOUNTED_YQ_BIN_PATH) except subprocess.CalledProcessError as exc: + logger.exception( + "Error installing yq, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise YQBuildError from exc @@ -429,31 +479,33 @@ def _disable_unattended_upgrades() -> None: try: # use subprocess run rather than operator-libs-linux's systemd library since the library # does not provide full features like mask. - subprocess.run( - ["/usr/bin/systemctl", "stop", APT_TIMER], check=True, timeout=30 - ) # nosec: B603 - subprocess.run( - ["/usr/bin/systemctl", "disable", APT_TIMER], check=True, timeout=30 + subprocess.check_output( + ["/usr/bin/systemctl", "stop", APT_TIMER], timeout=30 ) # nosec: B603 - subprocess.run( - ["/usr/bin/systemctl", "mask", APT_SVC], check=True, timeout=30 + subprocess.check_output( + ["/usr/bin/systemctl", "disable", APT_TIMER], timeout=30 ) # nosec: B603 - subprocess.run( - ["/usr/bin/systemctl", "stop", APT_UPGRADE_TIMER], check=True, timeout=30 + subprocess.check_output(["/usr/bin/systemctl", "mask", APT_SVC], timeout=30) # nosec: B603 + subprocess.check_output( + ["/usr/bin/systemctl", "stop", APT_UPGRADE_TIMER], timeout=30 ) # nosec: B603 - subprocess.run( # nosec: B603 - ["/usr/bin/systemctl", "disable", APT_UPGRADE_TIMER], check=True, timeout=30 + subprocess.check_output( # nosec: B603 + ["/usr/bin/systemctl", "disable", APT_UPGRADE_TIMER], timeout=30 ) - subprocess.run( - ["/usr/bin/systemctl", "mask", APT_UPGRAD_SVC], check=True, timeout=30 - ) # nosec: B603 - subprocess.run( - ["/usr/bin/systemctl", "daemon-reload"], check=True, timeout=30 + subprocess.check_output( + ["/usr/bin/systemctl", "mask", APT_UPGRAD_SVC], timeout=30 ) # nosec: B603 - subprocess.run( # nosec: B603 - ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], check=True, timeout=30 + subprocess.check_output(["/usr/bin/systemctl", "daemon-reload"], timeout=30) # nosec: B603 + subprocess.check_output( # nosec: B603 + ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], timeout=30 + ) + except subprocess.CalledProcessError as exc: + logger.exception( + "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, ) - except subprocess.SubprocessError as exc: raise UnattendedUpgradeDisableError from exc @@ -464,29 +516,33 @@ def _configure_system_users() -> None: SystemUserConfigurationError: If there was an error configuring ubuntu user. """ try: - subprocess.run( # nosec: B603 - ["/usr/sbin/useradd", "-m", UBUNTU_USER], check=True, timeout=30 + subprocess.check_output( # nosec: B603 + ["/usr/sbin/useradd", "-m", UBUNTU_USER], timeout=30 ) with (UBUNTU_HOME / ".profile").open("a") as profile_file: profile_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") with (UBUNTU_HOME / ".bashrc").open("a") as bashrc_file: bashrc_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") - subprocess.run( # nosec: B603 - ["/usr/sbin/groupadd", MICROK8S_GROUP], check=True, timeout=30 + subprocess.check_output(["/usr/sbin/groupadd", MICROK8S_GROUP], timeout=30) # nosec: B603 + subprocess.check_output( # nosec: B603 + ["/usr/sbin/usermod", "-aG", DOCKER_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.run( # nosec: B603 - ["/usr/sbin/usermod", "-aG", DOCKER_GROUP, UBUNTU_USER], check=True, timeout=30 + subprocess.check_output( # nosec: B603 + ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.run( # nosec: B603 - ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], check=True, timeout=30 + subprocess.check_output( # nosec: B603 + ["/usr/sbin/usermod", "-aG", LXD_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.run( # nosec: B603 - ["/usr/sbin/usermod", "-aG", LXD_GROUP, UBUNTU_USER], check=True, timeout=30 + subprocess.check_output( # nosec: B603 + ["/usr/bin/chmod", "777", "/usr/local/bin"], timeout=30 ) - subprocess.run( # nosec: B603 - ["/usr/bin/chmod", "777", "/usr/local/bin"], check=True, timeout=30 + except subprocess.CalledProcessError as exc: + logger.exception( + "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, ) - except subprocess.SubprocessError as exc: raise SystemUserConfigurationError from exc @@ -498,13 +554,19 @@ def _install_yarn() -> None: """ try: # 2024/04/26 There's a potential security risk here, npm is subject to toolchain attacks. - subprocess.run( - ["/usr/bin/npm", "install", "--global", "yarn"], check=True, timeout=60 * 5 + subprocess.check_output( + ["/usr/bin/npm", "install", "--global", "yarn"], timeout=60 * 5 ) # nosec: B603 - subprocess.run( - ["/usr/bin/npm", "cache", "clean", "--force"], check=True, timeout=60 + subprocess.check_output( + ["/usr/bin/npm", "cache", "clean", "--force"], timeout=60 ) # nosec: B603 - except subprocess.SubprocessError as exc: + except subprocess.CalledProcessError as exc: + logger.exception( + "Error installing Yarn, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise YarnInstallError from exc @@ -519,10 +581,15 @@ def _compress_image(image: Path) -> None: ImageCompressError: If there was something wrong compressing the image. """ try: - subprocess.run( # nosec: B603 + subprocess.check_output( # nosec: B603 ["/usr/bin/virt-sparsify", "--compress", str(image), str(IMAGE_OUTPUT_PATH)], - check=True, timeout=60 * 10, ) except subprocess.CalledProcessError as exc: + logger.exception( + "Error compressing image, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) raise ImageCompressError from exc diff --git a/src/github_runner_image_builder/chroot.py b/src/github_runner_image_builder/chroot.py index 1bd4658..5ef96d1 100644 --- a/src/github_runner_image_builder/chroot.py +++ b/src/github_runner_image_builder/chroot.py @@ -56,6 +56,7 @@ def __enter__(self) -> None: ["/usr/bin/mount", "--bind", f"/{shared_dir}", str(chroot_shared_dir)], check=True, timeout=30, + capture_output=True, ) except subprocess.CalledProcessError as exc: raise MountError from exc @@ -76,7 +77,7 @@ def __exit__(self, *_args: Any, **_kwargs: Any) -> None: os.close(cast(int, self.root)) try: - subprocess.run(["/usr/bin/sync"], check=True) # nosec: B603 + subprocess.run(["/usr/bin/sync"], check=True, capture_output=True) # nosec: B603 except subprocess.CalledProcessError as exc: raise SyncError from exc @@ -84,7 +85,7 @@ def __exit__(self, *_args: Any, **_kwargs: Any) -> None: chroot_shared_dir = self.chroot_path / shared_dir try: subprocess.run( - ["/usr/bin/umount", str(chroot_shared_dir)], check=True + ["/usr/bin/umount", str(chroot_shared_dir)], check=True, capture_output=True ) # nosec: B603 except subprocess.CalledProcessError as exc: raise MountError from exc @@ -93,6 +94,7 @@ def __exit__(self, *_args: Any, **_kwargs: Any) -> None: subprocess.run( # nosec: B603 ["/usr/bin/umount", "-l", str(self.chroot_path / CHROOT_DEVICE_DIR)], check=True, + capture_output=True, ) except subprocess.CalledProcessError as exc: raise MountError from exc diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index 9a6c21d..ce25181 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -117,7 +117,7 @@ def main(args: list[str] | None = None) -> None: store.get_latest_build_id( cloud_name=options.cloud_name, image_name=options.image_name ), - end=None, + end="", ) return diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index 816d31b..ee10ef6 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -134,6 +134,4 @@ async def test_get_image( image_id = openstack_connection.get_image_id(openstack_image_name) res = capsys.readouterr() - assert ( - res.out.strip() == image_id - ), f"Openstack image not matching, {res.out} {res.err}, {image_id}" + assert res.out == image_id, f"Openstack image not matching, {res.out} {res.err}, {image_id}" From d6cddec3b0e7d005f1fc849cc1c5a3a7eed24907 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 3 Jun 2024 02:10:29 +0000 Subject: [PATCH 21/63] log subprocess run outputs --- src/github_runner_image_builder/builder.py | 128 ++++++++++++++------- 1 file changed, 84 insertions(+), 44 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 9603b60..e2e11a9 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -115,19 +115,22 @@ def _install_dependencies() -> None: DependencyInstallError: If there was an error installing apt packages. """ try: - subprocess.check_output( + output = subprocess.check_output( ["/usr/bin/apt-get", "update", "-y"], encoding="utf-8", timeout=30 * 60 ) # nosec: B603 - subprocess.check_output( + logger.info("apt-get update out: %s", output) + output = subprocess.check_output( ["/usr/bin/apt-get", "install", "-y", "--no-install-recommends", *APT_DEPENDENCIES], encoding="utf-8", timeout=30 * 60, ) # nosec: B603 - subprocess.check_output( + logger.info("apt-get install out: %s", output) + output = subprocess.check_output( ["/usr/bin/snap", "install", SNAP_GO, "--classic"], encoding="utf-8", timeout=30 * 60, ) # nosec: B603 + logger.info("apt-get snap install go out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error installing dependencies, cmd: %s, code: %s, err: %s", @@ -145,7 +148,8 @@ def _enable_network_block_device() -> None: NetworkBlockDeviceError: If there was an error enable nbd kernel. """ try: - subprocess.check_output(["/usr/sbin/modprobe", "nbd"], timeout=10) # nosec: B603 + output = subprocess.check_output(["/usr/sbin/modprobe", "nbd"], timeout=10) # nosec: B603 + logger.info("modprobe nbd out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error enabling network block device, cmd: %s, code: %s, err: %s", @@ -190,14 +194,13 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: with ChrootContextManager(IMAGE_MOUNT_DIR): # operator_libs_linux apt package uses dpkg -l and that does not work well with # chroot env, hence use subprocess run. - subprocess.run( + output = subprocess.check_output( ["/usr/bin/apt-get", "update", "-y"], timeout=60 * 10, env={"DEBIAN_FRONTEND": "noninteractive"}, - check=True, - capture_output=True, ) # nosec: B603 - subprocess.run( # nosec: B603 + logger.info("apt-get update out: %s", output) + output = subprocess.check_output( # nosec: B603 [ "/usr/bin/apt-get", "install", @@ -207,9 +210,8 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: ], timeout=60 * 20, env={"DEBIAN_FRONTEND": "noninteractive"}, - check=True, - capture_output=True, ) + logger.info("apt-get install out: %s", output) _disable_unattended_upgrades() _configure_system_users() _install_yarn() @@ -234,51 +236,58 @@ def _clean_build_state() -> None: # The commands will fail if artefacts do not exist and hence there is no need to check the # output of subprocess runs. try: - subprocess.run( + output = subprocess.run( ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], timeout=30, check=False, - capture_output=True, ) # nosec: B603 - subprocess.run( + logger.info("umount dev out: %s", output) + output = subprocess.run( ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], timeout=30, check=False, capture_output=True, ) # nosec: B603 - subprocess.run( + logger.info("umount proc out: %s", output) + output = subprocess.run( ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], timeout=30, check=False, capture_output=True, ) # nosec: B603 - subprocess.run( + logger.info("umount sys out: %s", output) + output = subprocess.run( ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False, capture_output=True ) # nosec: B603 - subprocess.run( + logger.info("umount ubuntu-image out: %s", output) + output = subprocess.run( ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False, capture_output=True, ) # nosec: B603 - subprocess.run( # nosec: B603 + logger.info("umount nbd out: %s", output) + output = subprocess.run( # nosec: B603 ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False, capture_output=True, ) - subprocess.run( # nosec: B603 + logger.info("umount nbdp1 out: %s", output) + output = subprocess.run( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, check=False, capture_output=True, ) - subprocess.run( # nosec: B603 + logger.info("qemu-nbd disconnect nbd out: %s", output) + output = subprocess.run( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, check=False, capture_output=True, ) + logger.info("qemu-nbd disconnect nbdp1 out: %s", output) except subprocess.CalledProcessError as exc: raise CleanBuildStateError from exc @@ -353,10 +362,11 @@ def _resize_image(image_path: Path) -> None: ImageResizeError: If there was an error resizing the image. """ try: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-img", "resize", str(image_path), RESIZE_AMOUNT], timeout=60, ) + logger.info("qemu-img resize out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error resizing image, cmd: %s, code: %s, err: %s", @@ -376,7 +386,7 @@ def _replace_mounted_resolv_conf() -> None: @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _mount_network_block_device_partition() -> None: """Mount the network block device partition.""" - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 [ "/usr/bin/mount", "-o", @@ -386,6 +396,7 @@ def _mount_network_block_device_partition() -> None: ], timeout=60, ) + logger.info("mount nbd0p1 out: %s", output) def _mount_image_to_network_block_device(image_path: Path) -> None: @@ -398,10 +409,11 @@ def _mount_image_to_network_block_device(image_path: Path) -> None: ImageMountError: If there was an error mounting the image to network block device. """ try: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-nbd", f"--connect={NETWORK_BLOCK_DEVICE_PATH}", str(image_path)], timeout=60, ) + logger.info("qemu-nbd connect out: %s", output) _mount_network_block_device_partition() except subprocess.CalledProcessError as exc: logger.exception( @@ -420,13 +432,15 @@ def _resize_mount_partitions() -> None: ResizePartitionError: If there was an error resizing network block device partitions. """ try: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/growpart", str(NETWORK_BLOCK_DEVICE_PATH), "1"], timeout=10 * 60 ) - subprocess.check_output( # nosec: B603 + logger.info("growpart out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/sbin/resize2fs", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=10 * 60, ) + logger.info("resize2fs out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error resizing mount partitions, cmd: %s, code: %s, err: %s", @@ -445,19 +459,22 @@ def _install_yq() -> None: """ try: if not YQ_REPOSITORY_PATH.exists(): - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/git", "clone", str(YQ_REPOSITORY_URL), str(YQ_REPOSITORY_PATH)], timeout=60 * 10, ) + logger.info("git clone out: %s", output) else: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/git", "-C", str(YQ_REPOSITORY_PATH), "pull"], timeout=60 * 10, ) - subprocess.check_output( # nosec: B603 + logger.info("git pull out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/snap/bin/go", "build", "-C", str(YQ_REPOSITORY_PATH), "-o", str(HOST_YQ_BIN_PATH)], timeout=20 * 60, ) + logger.info("go build out: %s", output) shutil.copy(HOST_YQ_BIN_PATH, MOUNTED_YQ_BIN_PATH) except subprocess.CalledProcessError as exc: logger.exception( @@ -479,26 +496,38 @@ def _disable_unattended_upgrades() -> None: try: # use subprocess run rather than operator-libs-linux's systemd library since the library # does not provide full features like mask. - subprocess.check_output( + output = subprocess.check_output( ["/usr/bin/systemctl", "stop", APT_TIMER], timeout=30 ) # nosec: B603 - subprocess.check_output( + logger.info("systemctl stop apt timer out: %s", output) + output = subprocess.check_output( ["/usr/bin/systemctl", "disable", APT_TIMER], timeout=30 ) # nosec: B603 - subprocess.check_output(["/usr/bin/systemctl", "mask", APT_SVC], timeout=30) # nosec: B603 - subprocess.check_output( + logger.info("systemctl disable apt timer out: %s", output) + output = subprocess.check_output( + ["/usr/bin/systemctl", "mask", APT_SVC], timeout=30 + ) # nosec: B603 + logger.info("systemctl mask apt timer out: %s", output) + output = subprocess.check_output( ["/usr/bin/systemctl", "stop", APT_UPGRADE_TIMER], timeout=30 ) # nosec: B603 - subprocess.check_output( # nosec: B603 + logger.info("systemctl stop apt upgrade timer out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/bin/systemctl", "disable", APT_UPGRADE_TIMER], timeout=30 ) - subprocess.check_output( + logger.info("systemctl disable apt upgrade timer out: %s", output) + output = subprocess.check_output( ["/usr/bin/systemctl", "mask", APT_UPGRAD_SVC], timeout=30 ) # nosec: B603 - subprocess.check_output(["/usr/bin/systemctl", "daemon-reload"], timeout=30) # nosec: B603 - subprocess.check_output( # nosec: B603 + logger.info("systemctl mask apt upgrade timer out: %s", output) + output = subprocess.check_output( + ["/usr/bin/systemctl", "daemon-reload"], timeout=30 + ) # nosec: B603 + logger.info("systemctl daemon-reload out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], timeout=30 ) + logger.info("apt-get remove unattended-upgrades out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", @@ -516,26 +545,34 @@ def _configure_system_users() -> None: SystemUserConfigurationError: If there was an error configuring ubuntu user. """ try: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/sbin/useradd", "-m", UBUNTU_USER], timeout=30 ) + logger.info("useradd ubunutu out: %s", output) with (UBUNTU_HOME / ".profile").open("a") as profile_file: profile_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") with (UBUNTU_HOME / ".bashrc").open("a") as bashrc_file: bashrc_file.write(f"PATH=$PATH:{UBUNTU_HOME}/.local/bin\n") - subprocess.check_output(["/usr/sbin/groupadd", MICROK8S_GROUP], timeout=30) # nosec: B603 - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( + ["/usr/sbin/groupadd", MICROK8S_GROUP], timeout=30 + ) # nosec: B603 + logger.info("groupadd microk8s out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/sbin/usermod", "-aG", DOCKER_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.check_output( # nosec: B603 + logger.info("groupadd add ubuntu to docker group out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.check_output( # nosec: B603 + logger.info("groupadd add ubuntu to microk8s group out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/sbin/usermod", "-aG", LXD_GROUP, UBUNTU_USER], timeout=30 ) - subprocess.check_output( # nosec: B603 + logger.info("groupadd add ubuntu to lxd group out: %s", output) + output = subprocess.check_output( # nosec: B603 ["/usr/bin/chmod", "777", "/usr/local/bin"], timeout=30 ) + logger.info("chmod /usr/local/bin out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", @@ -554,12 +591,14 @@ def _install_yarn() -> None: """ try: # 2024/04/26 There's a potential security risk here, npm is subject to toolchain attacks. - subprocess.check_output( + output = subprocess.check_output( ["/usr/bin/npm", "install", "--global", "yarn"], timeout=60 * 5 ) # nosec: B603 - subprocess.check_output( + logger.info("npm install yarn out: %s", output) + output = subprocess.check_output( ["/usr/bin/npm", "cache", "clean", "--force"], timeout=60 ) # nosec: B603 + logger.info("npm cache clean out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error installing Yarn, cmd: %s, code: %s, err: %s", @@ -581,10 +620,11 @@ def _compress_image(image: Path) -> None: ImageCompressError: If there was something wrong compressing the image. """ try: - subprocess.check_output( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/virt-sparsify", "--compress", str(image), str(IMAGE_OUTPUT_PATH)], timeout=60 * 10, ) + logger.info("virt-sparsify compress out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error compressing image, cmd: %s, code: %s, err: %s", From 450123e202dad6abb46b0ebdff83c38d8c36e156 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 3 Jun 2024 09:01:03 +0000 Subject: [PATCH 22/63] test coverage --- tests/unit/test_builder.py | 74 +++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 1ab2f42..05e842e 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -39,9 +39,38 @@ ) -def test__install_dependencies_package_not_found(monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize( + "func, args", + [ + pytest.param("_install_dependencies", [], id="install dependencies"), + pytest.param("_enable_network_block_device", [], id="enable network block device"), + pytest.param("_resize_image", [MagicMock()], id="resize image"), + pytest.param("_resize_mount_partitions", [], id="resize mount partitions"), + pytest.param("_disable_unattended_upgrades", [], id="disable unattended upgrades"), + pytest.param("_configure_system_users", [], id="configure system users"), + pytest.param("_compress_image", [MagicMock()], id="compress image"), + ], +) +def test_subprocess_call_funcs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, func: str, args: list[Any] +): + """ + arrange: given functions that consist of subprocess calls only with mocked subprocess calls. + act: when the functions are called. + assert: no errors are raised. + """ + monkeypatch.setattr(subprocess, "check_output", MagicMock()) + monkeypatch.setattr(subprocess, "run", MagicMock()) + monkeypatch.setattr(builder, "UBUNTU_HOME", tmp_path) + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) + + assert getattr(builder, func)(*args) is None + + +def test__install_dependencies_error(monkeypatch: pytest.MonkeyPatch): """ - arrange: given apt.add_package that raises PackageNotFoundError. + arrange: given mocked subprocess.check_output calls that raises CalledProcessError. act: when _install_dependencies is called. assert: DependencyInstallError is raised. """ @@ -67,7 +96,7 @@ def test__enable_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): """ monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock(side_effect=subprocess.CalledProcessError(1, [], "Module nbd not found")), ) @@ -248,7 +277,7 @@ def test__resize_image_fail(monkeypatch: pytest.MonkeyPatch): ) monkeypatch.setattr( subprocess, - "run", + "check_output", mock_run, ) @@ -264,7 +293,7 @@ def test__mount_network_block_device_partition(monkeypatch: pytest.MonkeyPatch): act: when _mount_network_block_device_partition is called. assert: subprocess run call is made. """ - monkeypatch.setattr(subprocess, "run", (mock_run_call := MagicMock())) + monkeypatch.setattr(subprocess, "check_output", (mock_run_call := MagicMock())) builder._mount_network_block_device_partition() @@ -279,7 +308,7 @@ def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPat """ monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock(side_effect=subprocess.CalledProcessError(1, [], "", "error mounting")), ) @@ -296,7 +325,7 @@ def test__mount_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): act: when _mount_image_to_network_block_device is called. assert: expected calls are made. """ - monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) monkeypatch.setattr( builder, "_mount_network_block_device_partition", (mount_mock := MagicMock()) ) @@ -332,7 +361,7 @@ def test__resize_mount_partitions(monkeypatch: pytest.MonkeyPatch): """ monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock(side_effect=[None, subprocess.CalledProcessError(1, [], "", "resize error")]), ) @@ -350,7 +379,7 @@ def test__install_yq_error(monkeypatch: pytest.MonkeyPatch): """ monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock(side_effect=[None, subprocess.CalledProcessError(1, [], "", "Go build error.")]), ) @@ -367,7 +396,7 @@ def test__install_yq_already_exists(monkeypatch: pytest.MonkeyPatch): assert: Mock functions are called. """ monkeypatch.setattr(builder, "YQ_REPOSITORY_PATH", MagicMock(return_value=True)) - monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) builder._install_yq() @@ -382,7 +411,7 @@ def test__install_yq(monkeypatch: pytest.MonkeyPatch): act: when _install_yq is called. assert: Mock functions are called. """ - monkeypatch.setattr(subprocess, "run", (run_mock := MagicMock())) + monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) builder._install_yq() @@ -401,11 +430,11 @@ def test__disable_unattended_upgrades_subprocess_fail(monkeypatch: pytest.Monkey # pylint: disable=duplicate-code monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock( side_effect=[ *([None] * 7), - subprocess.SubprocessError("Failed to disable unattended upgrades"), + subprocess.CalledProcessError(1, [], "Failed to disable unattended upgrades", ""), ] ), ) @@ -425,8 +454,13 @@ def test__configure_system_users(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(builder, "UBUNTU_HOME", MagicMock()) monkeypatch.setattr( builder.subprocess, - "run", - MagicMock(side_effect=[*([None] * 5), subprocess.SubprocessError("Failed to add group.")]), + "check_output", + MagicMock( + side_effect=[ + *([None] * 5), + subprocess.CalledProcessError(1, [], "Failed to add group.", ""), + ] + ), ) with pytest.raises(SystemUserConfigurationError) as exc: @@ -444,11 +478,11 @@ def test__install_yarn_error(monkeypatch: pytest.MonkeyPatch): # The test mocks use similar codes. monkeypatch.setattr( # pylint: disable=duplicate-code subprocess, - "run", + "check_output", MagicMock( side_effect=[ None, - subprocess.CalledProcessError(1, [], "", "Failed to clean npm cache."), + subprocess.CalledProcessError(1, [], "Failed to clean npm cache.", ""), ] ), ) @@ -465,7 +499,7 @@ def test__install_yarn(monkeypatch: pytest.MonkeyPatch): act: when _install_yarn is called. assert: The function exists without raising an error. """ - monkeypatch.setattr(subprocess, "run", MagicMock()) + monkeypatch.setattr(subprocess, "check_output", MagicMock()) assert builder._install_yarn() is None @@ -480,7 +514,7 @@ def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr( subprocess, - "run", + "check_output", MagicMock(side_effect=subprocess.CalledProcessError(1, [], "Compression error")), ) @@ -537,7 +571,7 @@ def test_build_image_error( monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) monkeypatch.setattr(builder, "_install_yq", MagicMock()) monkeypatch.setattr(builder, "ChrootContextManager", MagicMock()) - monkeypatch.setattr(builder.subprocess, "run", MagicMock()) + monkeypatch.setattr(builder.subprocess, "check_output", MagicMock()) monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) monkeypatch.setattr(builder, "_install_yarn", MagicMock()) From df231fc51d84c02a72963a4ac6eca57132ddca37 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 3 Jun 2024 11:17:32 +0000 Subject: [PATCH 23/63] address comments --- src/github_runner_image_builder/builder.py | 133 ++++++++++++--------- src/github_runner_image_builder/cli.py | 2 +- src/github_runner_image_builder/config.py | 28 +---- src/github_runner_image_builder/errors.py | 20 ++-- src/github_runner_image_builder/store.py | 44 ++++--- tests/integration/conftest.py | 5 +- tests/unit/test_builder.py | 82 ++++++------- tests/unit/test_store.py | 27 +---- 8 files changed, 151 insertions(+), 190 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index e2e11a9..b2dde3a 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -17,7 +17,6 @@ from github_runner_image_builder.config import IMAGE_OUTPUT_PATH, Arch, BaseImage from github_runner_image_builder.errors import ( BaseImageDownloadError, - BuilderSetupError, BuildImageError, CleanBuildStateError, DependencyInstallError, @@ -26,6 +25,7 @@ ImageMountError, ImageResizeError, NetworkBlockDeviceError, + PermissionConfigurationError, ResizePartitionError, SystemUserConfigurationError, UnattendedUpgradeDisableError, @@ -45,6 +45,7 @@ "cloud-utils", # used for growpart. "golang-go", # used to build yq from source. ] +APT_NONINTERACTIVE_ENV = {"DEBIAN_FRONTEND": "noninteractive"} SNAP_GO = "go" # Constants for mounting images @@ -58,10 +59,6 @@ MOUNTED_RESOLV_CONF_PATH = IMAGE_MOUNT_DIR / "etc/resolv.conf" HOST_RESOLV_CONF_PATH = Path("/etc/resolv.conf") -# Constants for chroot environment Python symmlinks -DEFAULT_PYTHON_PATH = Path("/usr/bin/python3") -SYM_LINK_PYTHON_PATH = Path("/usr/bin/python") - # Constants for disabling automatic apt updates APT_TIMER = "apt-daily.timer" APT_SVC = "apt-daily.service" @@ -94,18 +91,11 @@ def initialize() -> None: - """Configure the host machine to build images. - - Raises: - BuilderSetupError: If there was an error setting up the host device for building images. - """ - try: - logger.info("Installing dependencies.") - _install_dependencies() - logger.info("Enabling network block device.") - _enable_network_block_device() - except ImageBuilderBaseError as exc: - raise BuilderSetupError from exc + """Configure the host machine to build images.""" + logger.info("Installing dependencies.") + _install_dependencies() + logger.info("Enabling network block device.") + _enable_network_block_device() def _install_dependencies() -> None: @@ -116,12 +106,16 @@ def _install_dependencies() -> None: """ try: output = subprocess.check_output( - ["/usr/bin/apt-get", "update", "-y"], encoding="utf-8", timeout=30 * 60 + ["/usr/bin/apt-get", "update", "-y"], + encoding="utf-8", + env=APT_NONINTERACTIVE_ENV, + timeout=30 * 60, ) # nosec: B603 logger.info("apt-get update out: %s", output) output = subprocess.check_output( ["/usr/bin/apt-get", "install", "-y", "--no-install-recommends", *APT_DEPENDENCIES], encoding="utf-8", + env=APT_NONINTERACTIVE_ENV, timeout=30 * 60, ) # nosec: B603 logger.info("apt-get install out: %s", output) @@ -130,7 +124,7 @@ def _install_dependencies() -> None: encoding="utf-8", timeout=30 * 60, ) # nosec: B603 - logger.info("apt-get snap install go out: %s", output) + logger.info("snap install go out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error installing dependencies, cmd: %s, code: %s, err: %s", @@ -180,8 +174,6 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: _resize_image(image_path=base_image_path) logger.info("Mounting network block device.") _mount_image_to_network_block_device(image_path=base_image_path) - logger.info("Replacing resolv.conf.") - _replace_mounted_resolv_conf() logger.info("Resizing partitions.") _resize_mount_partitions() logger.info("Installing YQ from source.") @@ -191,13 +183,15 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: try: logger.info("Setting up chroot environment.") + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() with ChrootContextManager(IMAGE_MOUNT_DIR): # operator_libs_linux apt package uses dpkg -l and that does not work well with # chroot env, hence use subprocess run. output = subprocess.check_output( ["/usr/bin/apt-get", "update", "-y"], timeout=60 * 10, - env={"DEBIAN_FRONTEND": "noninteractive"}, + env=APT_NONINTERACTIVE_ENV, ) # nosec: B603 logger.info("apt-get update out: %s", output) output = subprocess.check_output( # nosec: B603 @@ -209,11 +203,12 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: *IMAGE_DEFAULT_APT_PACKAGES, ], timeout=60 * 20, - env={"DEBIAN_FRONTEND": "noninteractive"}, + env=APT_NONINTERACTIVE_ENV, ) logger.info("apt-get install out: %s", output) _disable_unattended_upgrades() _configure_system_users() + _configure_usr_local_bin() _install_yarn() except ChrootBaseError as exc: raise BuildImageError from exc @@ -223,7 +218,7 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: try: logger.info("Compressing image.") _compress_image(base_image_path) - except ImageBuilderBaseError as exc: + except ImageCompressError as exc: raise BuildImageError from exc @@ -288,7 +283,7 @@ def _clean_build_state() -> None: capture_output=True, ) logger.info("qemu-nbd disconnect nbdp1 out: %s", output) - except subprocess.CalledProcessError as exc: + except subprocess.SubprocessError as exc: raise CleanBuildStateError from exc @@ -377,28 +372,6 @@ def _resize_image(image_path: Path) -> None: raise ImageResizeError from exc -def _replace_mounted_resolv_conf() -> None: - """Replace resolv.conf to host resolv.conf to allow networking.""" - MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) - shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) - - -@retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) -def _mount_network_block_device_partition() -> None: - """Mount the network block device partition.""" - output = subprocess.check_output( # nosec: B603 - [ - "/usr/bin/mount", - "-o", - "rw", - str(NETWORK_BLOCK_DEVICE_PARTITION_PATH), - str(IMAGE_MOUNT_DIR), - ], - timeout=60, - ) - logger.info("mount nbd0p1 out: %s", output) - - def _mount_image_to_network_block_device(image_path: Path) -> None: """Mount the image to network block device in preparation for chroot. @@ -425,6 +398,23 @@ def _mount_image_to_network_block_device(image_path: Path) -> None: raise ImageMountError from exc +# Network block device may fail to mount, retrying will usually fix this. +@retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) +def _mount_network_block_device_partition() -> None: + """Mount the network block device partition.""" + output = subprocess.check_output( # nosec: B603 + [ + "/usr/bin/mount", + "-o", + "rw", + str(NETWORK_BLOCK_DEVICE_PARTITION_PATH), + str(IMAGE_MOUNT_DIR), + ], + timeout=60, + ) + logger.info("mount nbd0p1 out: %s", output) + + def _resize_mount_partitions() -> None: """Resize the block partition to fill available space. @@ -486,6 +476,12 @@ def _install_yq() -> None: raise YQBuildError from exc +def _replace_mounted_resolv_conf() -> None: + """Replace resolv.conf to host resolv.conf to allow networking.""" + MOUNTED_RESOLV_CONF_PATH.unlink(missing_ok=True) + shutil.copy(str(HOST_RESOLV_CONF_PATH), str(MOUNTED_RESOLV_CONF_PATH)) + + def _disable_unattended_upgrades() -> None: """Disable unatteneded upgrades to prevent apt locks. @@ -525,7 +521,9 @@ def _disable_unattended_upgrades() -> None: ) # nosec: B603 logger.info("systemctl daemon-reload out: %s", output) output = subprocess.check_output( # nosec: B603 - ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], timeout=30 + ["/usr/bin/apt-get", "remove", "-y", "unattended-upgrades"], + env=APT_NONINTERACTIVE_ENV, + timeout=30, ) logger.info("apt-get remove unattended-upgrades out: %s", output) except subprocess.CalledProcessError as exc: @@ -558,29 +556,45 @@ def _configure_system_users() -> None: ) # nosec: B603 logger.info("groupadd microk8s out: %s", output) output = subprocess.check_output( # nosec: B603 - ["/usr/sbin/usermod", "-aG", DOCKER_GROUP, UBUNTU_USER], timeout=30 - ) - logger.info("groupadd add ubuntu to docker group out: %s", output) - output = subprocess.check_output( # nosec: B603 - ["/usr/sbin/usermod", "-aG", MICROK8S_GROUP, UBUNTU_USER], timeout=30 + [ + "/usr/sbin/usermod", + "-aG", + f"{DOCKER_GROUP},{MICROK8S_GROUP},{LXD_GROUP}", + UBUNTU_USER, + ], + timeout=30, ) - logger.info("groupadd add ubuntu to microk8s group out: %s", output) - output = subprocess.check_output( # nosec: B603 - ["/usr/sbin/usermod", "-aG", LXD_GROUP, UBUNTU_USER], timeout=30 + logger.info("usrmod to ubuntu user out: %s", output) + except subprocess.CalledProcessError as exc: + logger.exception( + "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, ) - logger.info("groupadd add ubuntu to lxd group out: %s", output) + raise SystemUserConfigurationError from exc + + +def _configure_usr_local_bin() -> None: + """Change the permissions of /usr/local/bin dir to match GH hosted runners permissions. + + Raises: + PermissionConfigurationError: if there was an error changing permissions. + """ + try: + # The 777 is to match the behavior of GitHub hosted runners output = subprocess.check_output( # nosec: B603 ["/usr/bin/chmod", "777", "/usr/local/bin"], timeout=30 ) logger.info("chmod /usr/local/bin out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( - "Error disabling unattended upgrades, cmd: %s, code: %s, err: %s", + "Error changing /usr/local/bin/ permissions, cmd: %s, code: %s, err: %s", exc.cmd, exc.returncode, exc.output, ) - raise SystemUserConfigurationError from exc + raise PermissionConfigurationError from exc def _install_yarn() -> None: @@ -609,6 +623,7 @@ def _install_yarn() -> None: raise YarnInstallError from exc +# Image compression might fail for arbitrary reasons - retrying usually solves this. @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _compress_image(image: Path) -> None: """Compress the image. diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index ce25181..f7fb279 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -193,4 +193,4 @@ def _build_and_upload( ) if callback_script_path: # The callback script is a user trusted script. - subprocess.check_call(["/bin/bash", str(callback_script_path), image_id]) # nosec: B603 + subprocess.check_call([str(callback_script_path), image_id]) # nosec: B603 diff --git a/src/github_runner_image_builder/config.py b/src/github_runner_image_builder/config.py index 6527661..0f91e77 100644 --- a/src/github_runner_image_builder/config.py +++ b/src/github_runner_image_builder/config.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Literal +from github_runner_image_builder.errors import UnsupportedArchitectureError + logger = logging.getLogger(__name__) ACTION_INIT = "init" @@ -51,30 +53,6 @@ class Arch(str, Enum): X64 = "x64" -class UnsupportedArchitectureError(Exception): - """Raised when given machine architecture is unsupported. - - Attributes: - arch: The current machine architecture. - """ - - def __str__(self) -> str: - """Represent the error in string format. - - Returns: - The error in string format. - """ - return f"UnsupportedArchitectureError: {self.arch}" - - def __init__(self, arch: str) -> None: - """Initialize a new instance of the UnsupportedArchitectureError exception. - - Args: - arch: The current machine architecture. - """ - self.arch = arch - - ARCHITECTURES_ARM64 = {"aarch64", "arm64"} ARCHITECTURES_X86 = {"x86_64"} @@ -95,7 +73,7 @@ def get_supported_arch() -> Arch: case arch if arch in ARCHITECTURES_X86: return Arch.X64 case _: - raise UnsupportedArchitectureError(arch=arch) + raise UnsupportedArchitectureError() class BaseImage(str, Enum): diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py index 428577a..05aff57 100644 --- a/src/github_runner_image_builder/errors.py +++ b/src/github_runner_image_builder/errors.py @@ -8,19 +8,19 @@ class ImageBuilderBaseError(Exception): """Represents an error with any builder related executions.""" +class BuilderSetupError(ImageBuilderBaseError): + """Represents an error while setting up host machine as builder.""" + + # nosec: B603: All subprocess runs are run with trusted executables. -class DependencyInstallError(ImageBuilderBaseError): +class DependencyInstallError(BuilderSetupError): """Represents an error while installing required dependencies.""" -class NetworkBlockDeviceError(ImageBuilderBaseError): +class NetworkBlockDeviceError(BuilderSetupError): """Represents an error while enabling network block device.""" -class BuilderSetupError(ImageBuilderBaseError): - """Represents an error while setting up host machine as builder.""" - - class UnsupportedArchitectureError(ImageBuilderBaseError): """Raised when given machine architecture is unsupported.""" @@ -53,6 +53,10 @@ class SystemUserConfigurationError(ImageBuilderBaseError): """Represents an error while adding user to chroot env.""" +class PermissionConfigurationError(ImageBuilderBaseError): + """Represents an error while modifying dir permissions.""" + + class YQBuildError(ImageBuilderBaseError): """Represents an error while building yq binary from source.""" @@ -81,9 +85,5 @@ class UploadImageError(OpenstackBaseError): """Represents an error when uploading image to Openstack.""" -class GetImageError(OpenstackBaseError): - """Represents an error when fetching images from Openstack.""" - - class OpenstackError(OpenstackBaseError): """Represents an error while communicating with Openstack.""" diff --git a/src/github_runner_image_builder/store.py b/src/github_runner_image_builder/store.py index ac249a5..7cbaed5 100644 --- a/src/github_runner_image_builder/store.py +++ b/src/github_runner_image_builder/store.py @@ -12,7 +12,7 @@ import openstack.exceptions from openstack.image.v2.image import Image -from github_runner_image_builder.errors import GetImageError, OpenstackError, UploadImageError +from github_runner_image_builder.errors import OpenstackError, UploadImageError logger = logging.getLogger(__name__) @@ -59,6 +59,8 @@ def _prune_old_images( num_revisions: The number of revisions to keep. """ images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) + if not images: + return images_to_prune = images[num_revisions:] for image in images_to_prune: try: @@ -69,6 +71,23 @@ def _prune_old_images( continue +def get_latest_build_id(cloud_name: str, image_name: str) -> str | None: + """Fetch the latest image id. + + Args: + cloud_name: The Openstack cloud to use from clouds.yaml. + image_name: The image name to search for. + + Returns: + The image ID if exists, None otherwise. + """ + with openstack.connect(cloud=cloud_name) as connection: + images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) + if not images: + return None + return images[0].id + + def _get_sorted_images_by_created_at( connection: openstack.connection.Connection, image_name: str ) -> list[Image]: @@ -90,26 +109,3 @@ def _get_sorted_images_by_created_at( raise OpenstackError from exc return sorted(images, key=lambda image: image.created_at, reverse=True) - - -def get_latest_build_id(cloud_name: str, image_name: str) -> str | None: - """Fetch the latest image id. - - Args: - cloud_name: The Openstack cloud to use from clouds.yaml. - image_name: The image name to search for. - - Raises: - GetImageError: If there was an error fetching image from Openstack. - - Returns: - The image ID if exists, None otherwise. - """ - with openstack.connect(cloud=cloud_name) as connection: - try: - images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) - except OpenstackError as exc: - raise GetImageError from exc - if not images: - return None - return images[0].id diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0c3e6ab..700e0d3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -10,6 +10,7 @@ import pytest import yaml from openstack.connection import Connection +from openstack.image.v2.image import Image from github_runner_image_builder.cli import main @@ -150,4 +151,6 @@ def cli_run_fixture( yield - openstack_connection.delete_image(openstack_image_name) + openstack_image: Image + for openstack_image in openstack_connection.search_images(openstack_image_name): + openstack_connection.delete_image(openstack_image.id) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 05e842e..3489ff1 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -18,7 +18,6 @@ Arch, BaseImage, BaseImageDownloadError, - BuilderSetupError, BuildImageError, ChrootBaseError, CleanBuildStateError, @@ -27,6 +26,7 @@ ImageMountError, ImageResizeError, NetworkBlockDeviceError, + PermissionConfigurationError, ResizePartitionError, SupportedBaseImageArch, SystemUserConfigurationError, @@ -48,6 +48,7 @@ pytest.param("_resize_mount_partitions", [], id="resize mount partitions"), pytest.param("_disable_unattended_upgrades", [], id="disable unattended upgrades"), pytest.param("_configure_system_users", [], id="configure system users"), + pytest.param("_configure_usr_local_bin", [], id="configure /usr/local/bin"), pytest.param("_compress_image", [MagicMock()], id="compress image"), ], ) @@ -68,6 +69,21 @@ def test_subprocess_call_funcs( assert getattr(builder, func)(*args) is None +def test_initialize(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given sub functions of initialize. + act: when initialize is called. + assert: the subfunctions are called. + """ + monkeypatch.setattr(builder, "_install_dependencies", (install_mock := MagicMock())) + monkeypatch.setattr(builder, "_enable_network_block_device", (enable_nbd_mock := MagicMock())) + + builder.initialize() + + install_mock.assert_called() + enable_nbd_mock.assert_called() + + def test__install_dependencies_error(monkeypatch: pytest.MonkeyPatch): """ arrange: given mocked subprocess.check_output calls that raises CalledProcessError. @@ -106,48 +122,6 @@ def test__enable_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): assert "Module nbd not found" in str(exc.getrepr()) -@pytest.mark.parametrize( - "patch_obj, sub_func, exception, expected_message", - [ - pytest.param( - builder, - "_install_dependencies", - DependencyInstallError("Dependency not found"), - "Dependency not found", - id="Dependency not found", - ), - pytest.param( - builder, - "_enable_network_block_device", - NetworkBlockDeviceError("Unable to enable nbd"), - "Unable to enable nbd", - id="Failed to enable nbd", - ), - ], -) -def test_setup_builder_fail( - patch_obj: Any, - sub_func: str, - exception: Exception, - expected_message: str, - monkeypatch: pytest.MonkeyPatch, -): - """ - arrange: given a monkeypatched sub functions of setup_builder that raises given exceptions. - act: when setup_builder is called. - assert: A BuilderSetupError is raised. - """ - mock_func = MagicMock(side_effect=exception) - monkeypatch.setattr(builder, "_install_dependencies", MagicMock) - monkeypatch.setattr(builder, "_enable_network_block_device", MagicMock) - monkeypatch.setattr(patch_obj, sub_func, mock_func) - - with pytest.raises(BuilderSetupError) as exc: - builder.initialize() - - assert expected_message in str(exc.getrepr()) - - def test__get_supported_runner_arch_unsupported_error(): """ arrange: given an architecture value that isn't supported. @@ -457,7 +431,7 @@ def test__configure_system_users(monkeypatch: pytest.MonkeyPatch): "check_output", MagicMock( side_effect=[ - *([None] * 5), + *([None] * 2), subprocess.CalledProcessError(1, [], "Failed to add group.", ""), ] ), @@ -469,6 +443,26 @@ def test__configure_system_users(monkeypatch: pytest.MonkeyPatch): assert "Failed to add group." in str(exc.getrepr()) +def test__configure_usr_local_bin(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched subprocess run calls that raises an exception. + act: when _configure_usr_local_bin is called. + assert: PermissionConfigurationError is raised. + """ + monkeypatch.setattr( + builder.subprocess, + "check_output", + MagicMock( + side_effect=subprocess.CalledProcessError(1, [], "Failed change permissions.", ""), + ), + ) + + with pytest.raises(PermissionConfigurationError) as exc: + builder._configure_usr_local_bin() + + assert "Failed change permissions." in str(exc.getrepr()) + + def test__install_yarn_error(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched subprocess.run that raises an error. diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index 1c68ba9..fa044c4 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -12,13 +12,7 @@ from openstack.connection import Connection from github_runner_image_builder import store -from github_runner_image_builder.store import ( - GetImageError, - Image, - OpenstackError, - UploadImageError, - openstack, -) +from github_runner_image_builder.store import Image, OpenstackError, UploadImageError, openstack from tests.unit.factories import MockOpenstackImageFactory @@ -159,25 +153,6 @@ def test_upload_image(mock_connection: MagicMock): ) -@pytest.mark.usefixtures("mock_connection") -def test_get_latest_image_id_error(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a mocked _get_images_by_latest function that raises an exception. - act: when get_latest_image_id is called. - assert: GetImageError is raised. - """ - monkeypatch.setattr( - store, - "_get_sorted_images_by_created_at", - MagicMock(side_effect=OpenstackError("Unauthorized")), - ) - - with pytest.raises(GetImageError) as exc: - store.get_latest_build_id(cloud_name=MagicMock(), image_name=MagicMock()) - - assert "Unauthorized" in str(exc.getrepr()) - - @pytest.mark.usefixtures("mock_connection") @pytest.mark.parametrize( "images, expected_id", From 6e5b90fa775afc322b8cb84eb4807c31e39da61d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 3 Jun 2024 13:46:02 +0000 Subject: [PATCH 24/63] make script executable --- tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 700e0d3..ca2c446 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -116,6 +116,7 @@ def callback_script_fixture(callback_result_path: Path) -> Path: """, encoding="utf-8", ) + callback_script.chmod(0o775) return callback_script From d10ef204c2808d2e4f76b8fd066a3b59b76b8654 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 02:37:08 +0000 Subject: [PATCH 25/63] refactor args --- src/github_runner_image_builder/builder.py | 43 ++- src/github_runner_image_builder/cli.py | 70 +++-- src/github_runner_image_builder/errors.py | 36 +-- tests/unit/test_cli.py | 315 ++++++++++++++------- 4 files changed, 287 insertions(+), 177 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index b2dde3a..f3debfb 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -20,7 +20,6 @@ BuildImageError, CleanBuildStateError, DependencyInstallError, - ImageBuilderBaseError, ImageCompressError, ImageMountError, ImageResizeError, @@ -165,26 +164,23 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: BuildImageError: If there was an error building the image. """ IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) + logger.info("Cleaning build state.") + _clean_build_state() + logger.info("Downloading base image.") + base_image_path = _download_base_image(arch=arch, base_image=base_image) + logger.info("Resizing base image.") + _resize_image(image_path=base_image_path) + logger.info("Mounting network block device.") + _mount_image_to_network_block_device(image_path=base_image_path) + logger.info("Resizing partitions.") + _resize_mount_partitions() + logger.info("Installing YQ from source.") + _install_yq() + + logger.info("Setting up chroot environment.") + logger.info("Replacing resolv.conf.") + _replace_mounted_resolv_conf() try: - logger.info("Cleaning build state.") - _clean_build_state() - logger.info("Downloading base image.") - base_image_path = _download_base_image(arch=arch, base_image=base_image) - logger.info("Resizing base image.") - _resize_image(image_path=base_image_path) - logger.info("Mounting network block device.") - _mount_image_to_network_block_device(image_path=base_image_path) - logger.info("Resizing partitions.") - _resize_mount_partitions() - logger.info("Installing YQ from source.") - _install_yq() - except ImageBuilderBaseError as exc: - raise BuildImageError from exc - - try: - logger.info("Setting up chroot environment.") - logger.info("Replacing resolv.conf.") - _replace_mounted_resolv_conf() with ChrootContextManager(IMAGE_MOUNT_DIR): # operator_libs_linux apt package uses dpkg -l and that does not work well with # chroot env, hence use subprocess run. @@ -215,11 +211,8 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: finally: _clean_build_state() - try: - logger.info("Compressing image.") - _compress_image(base_image_path) - except ImageCompressError as exc: - raise BuildImageError from exc + logger.info("Compressing image.") + _compress_image(base_image_path) def _clean_build_state() -> None: diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index f7fb279..f8425a1 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -7,7 +7,6 @@ # Subprocess module is used to execute trusted commands import subprocess # nosec: B404 -import sys from pathlib import Path from typing import cast @@ -29,11 +28,43 @@ def main(args: list[str] | None = None) -> None: Args: args: Command line arguments. + + Raises: + ValueError: If invalid action argument was supplied. """ - # The following line is used for unit testing. - if args is None: # pragma: nocover - args = sys.argv[1:] + options = _parse_args(args) + if options.action == "init": + builder.initialize() + return + if options.action == "latest-build-id": + print( + store.get_latest_build_id( + cloud_name=options.cloud_name, image_name=options.image_name + ), + end="", + ) + return + if options.action == "run": + _build_and_upload( + base=options.base, + cloud_name=options.cloud_name, + image_name=options.image_name, + keep_revisions=options.keep_revisions, + callback_script_path=options.callback_script_path, + ) + return + raise ValueError("Invalid CLI action argument.") + + +def _parse_args(args: list[str] | None = None) -> ActionsNamespace: + """Parse CLI arguments. + + Args: + args: Command line arguments. + Returns: + An object with parsed user inputs. + """ parser = argparse.ArgumentParser( prog="Github runner image builder CLI", description="Builds github runner image and uploads it to openstack.", @@ -55,12 +86,12 @@ def main(args: list[str] | None = None) -> None: "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " "of the following order: current directory, ~/.config/openstack, /etc/openstack." ), - type=non_empty_string, + type=_non_empty_string, ) get_latest_id_parser.add_argument( dest="image_name", help="The image name uploaded to Openstack.", - type=non_empty_string, + type=_non_empty_string, ) run_parser.add_argument( dest="cloud_name", @@ -68,12 +99,12 @@ def main(args: list[str] | None = None) -> None: "The cloud to use from the clouds.yaml file. The CLI looks for clouds.yaml in paths " "of the following order: current directory, ~/.config/openstack, /etc/openstack." ), - type=non_empty_string, + type=_non_empty_string, ) run_parser.add_argument( dest="image_name", help="The image name to upload to Openstack.", - type=non_empty_string, + type=_non_empty_string, ) run_parser.add_argument( "-b", @@ -108,26 +139,7 @@ def main(args: list[str] | None = None) -> None: ), ) options = cast(ActionsNamespace, parser.parse_args(args)) - if options.action == "init": - builder.initialize() - return - - if options.action == "latest-build-id": - print( - store.get_latest_build_id( - cloud_name=options.cloud_name, image_name=options.image_name - ), - end="", - ) - return - - _build_and_upload( - base=options.base, - cloud_name=options.cloud_name, - image_name=options.image_name, - keep_revisions=options.keep_revisions, - callback_script_path=options.callback_script_path, - ) + return options def _existing_path(value: str) -> Path: @@ -148,7 +160,7 @@ def _existing_path(value: str) -> Path: return path -def non_empty_string(arg: str) -> str: +def _non_empty_string(arg: str) -> str: """Check that the argument is non-empty. Args: diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py index 05aff57..27d0922 100644 --- a/src/github_runner_image_builder/errors.py +++ b/src/github_runner_image_builder/errors.py @@ -8,16 +8,16 @@ class ImageBuilderBaseError(Exception): """Represents an error with any builder related executions.""" -class BuilderSetupError(ImageBuilderBaseError): +class BuilderInitializeError(ImageBuilderBaseError): """Represents an error while setting up host machine as builder.""" # nosec: B603: All subprocess runs are run with trusted executables. -class DependencyInstallError(BuilderSetupError): +class DependencyInstallError(BuilderInitializeError): """Represents an error while installing required dependencies.""" -class NetworkBlockDeviceError(BuilderSetupError): +class NetworkBlockDeviceError(BuilderInitializeError): """Represents an error while enabling network block device.""" @@ -25,54 +25,54 @@ class UnsupportedArchitectureError(ImageBuilderBaseError): """Raised when given machine architecture is unsupported.""" -class CleanBuildStateError(ImageBuilderBaseError): +class BuildImageError(ImageBuilderBaseError): + """Represents an error while building the image.""" + + +class CleanBuildStateError(BuildImageError): """Represents an error cleaning up build state.""" -class BaseImageDownloadError(ImageBuilderBaseError): +class BaseImageDownloadError(BuildImageError): """Represents an error downloading base image.""" -class ImageResizeError(ImageBuilderBaseError): +class ImageResizeError(BuildImageError): """Represents an error while resizing the image.""" -class ImageMountError(ImageBuilderBaseError): +class ImageMountError(BuildImageError): """Represents an error while mounting the image to network block device.""" -class ResizePartitionError(ImageBuilderBaseError): +class ResizePartitionError(BuildImageError): """Represents an error while resizing network block device partitions.""" -class UnattendedUpgradeDisableError(ImageBuilderBaseError): +class UnattendedUpgradeDisableError(BuildImageError): """Represents an error while disabling unattended-upgrade related services.""" -class SystemUserConfigurationError(ImageBuilderBaseError): +class SystemUserConfigurationError(BuildImageError): """Represents an error while adding user to chroot env.""" -class PermissionConfigurationError(ImageBuilderBaseError): +class PermissionConfigurationError(BuildImageError): """Represents an error while modifying dir permissions.""" -class YQBuildError(ImageBuilderBaseError): +class YQBuildError(BuildImageError): """Represents an error while building yq binary from source.""" -class YarnInstallError(ImageBuilderBaseError): +class YarnInstallError(BuildImageError): """Represents an error installilng Yarn.""" -class ImageCompressError(ImageBuilderBaseError): +class ImageCompressError(BuildImageError): """Represents an error while compressing cloud-img.""" -class BuildImageError(ImageBuilderBaseError): - """Represents an error while building the image.""" - - class OpenstackBaseError(Exception): """Represents an error while interacting with Openstack.""" diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 8ea7aa7..1bbe158 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -13,7 +13,6 @@ import pytest from github_runner_image_builder import cli -from github_runner_image_builder.cli import main @pytest.fixture(scope="function", name="callback_path") @@ -36,129 +35,113 @@ def run_inputs_fixture(callback_path: Path): } -def test__existing_path(tmp_path: Path): +def test_main_invalid_choice(monkeypatch: pytest.MonkeyPatch): """ - arrange: given a path that does not exist. - act: when _existing_path is called. + arrange: given a monkeypatched _parse_args that returns invalid action choice. + act: when main is called. assert: ValueError is raised. """ - not_exists_path = tmp_path / "non-existent" + monkeypatch.setattr( + cli, "_parse_args", MagicMock(return_value=cli.ActionsNamespace(action="invalid")) + ) + with pytest.raises(ValueError) as exc: - cli._existing_path(str(not_exists_path)) + cli.main() - assert f"Given path {not_exists_path} not found." in str(exc.getrepr()) + assert "Invalid CLI action argument." in str(exc.getrepr()) @pytest.mark.parametrize( - "callback_script", + "action", [ - pytest.param(None, id="No callback script"), - pytest.param(Path("tmp_path"), id="Callback script"), + pytest.param("init", id="init"), + pytest.param("latest-build-id", id="latest-build-id"), + pytest.param("run", id="run"), ], ) -def test__build_and_upload(monkeypatch: pytest.MonkeyPatch, callback_script: Path | None): +def test_main(monkeypatch: pytest.MonkeyPatch, action: str): """ - arrange: given a monkeypatched builder.setup_builder function. - act: when _build is called. - assert: the mock function is called. + arrange: given a monkeypatched _parse_args that returns valid choice. + act: when main is called. + assert: mocked subfunctions are called correctly. """ - monkeypatch.setattr(cli.builder, "build_image", (builder_mock := MagicMock())) - monkeypatch.setattr(cli.store, "upload_image", MagicMock(return_value="test-image-id")) - monkeypatch.setattr(cli.subprocess, "check_call", MagicMock()) + actions_namespace_mock = MagicMock(autospec=cli.ActionsNamespace) + actions_namespace_mock.action = action + monkeypatch.setattr(cli, "_parse_args", MagicMock(return_value=actions_namespace_mock)) + monkeypatch.setattr(cli.builder, "initialize", init_mock := MagicMock()) + monkeypatch.setattr(cli.store, "get_latest_build_id", latest_build_mock := MagicMock()) + monkeypatch.setattr(cli, "_build_and_upload", build_mock := MagicMock()) - cli._build_and_upload( - base="jammy", - cloud_name=MagicMock(), - image_name=MagicMock(), - keep_revisions=MagicMock(), - callback_script_path=callback_script, - ) + cli.main() - builder_mock.assert_called_once() + assert any((init_mock.called, latest_build_mock.called, build_mock.called)) @pytest.mark.parametrize( - "choice", + "invalid_action", [ - pytest.param("", id="no choice"), - pytest.param("invalid", id="invalid choice"), + pytest.param("", id="empty"), + pytest.param("invalid", id="invalid"), ], ) -def test_main_invalid_choice(monkeypatch: pytest.MonkeyPatch, choice: str): +def test__parse_args_invalid_action(invalid_action: str): """ - arrange: given invalid argument choice and mocked builder functions. - act: when main is called. - assert: SystemExit is raised and mocked builder functions are not called. - """ - monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) - monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) - monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - - with pytest.raises(SystemExit): - main([choice]) - - initialize_mock.assert_not_called() - get_mock.assert_not_called() - build_mock.assert_not_called() - - -def test_main_init(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given init argument and mocked builder functions. - act: when main is called. - assert: initialize builder mock function is called. + arrange: given invalid action arguments. + act: when _parse_args is called. + assert: SystemExit error is raised. """ - monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) - monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) - monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - - main(["init"]) + with pytest.raises(SystemExit) as exc: + cli._parse_args([invalid_action]) - initialize_mock.assert_called() - get_mock.assert_not_called() - build_mock.assert_not_called() + assert "invalid choice" in str(exc.getrepr()) -def test_main_latest_build_id(monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize( + "invalid_args", + [ + pytest.param({"": ""}, id="empty cloud name positional argument"), + pytest.param({" ": ""}, id="empty image name positional argument"), + ], +) +def test__parse_args_invalid_latest_build_id_args(run_inputs: dict, invalid_args: dict): """ - arrange: given latest-build-id argument and mocked builder functions. - act: when main is called. - assert: get mock function is called. + arrange: given invalid latest-build-id action arguments. + act: when _parse_args is called. + assert: SystemExit error is raised. """ - monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) - monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) - monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) + run_inputs.update(invalid_args) + inputs = list( + # if flag does not exist, append it as a positional argument. + itertools.chain.from_iterable( + (flag, value) if flag.strip() else (value,) for (flag, value) in run_inputs.items() + ) + ) - main(["latest-build-id", "test-cloud", "test-image"]) + with pytest.raises(SystemExit) as exc: + cli._parse_args(inputs) - initialize_mock.assert_not_called() - get_mock.assert_called() - build_mock.assert_not_called() + assert "invalid choice" in str(exc.getrepr()) @pytest.mark.parametrize( - "invalid_patch", + "invalid_args", [ pytest.param({"--base-image": ""}, id="no base-image"), pytest.param({"--base-image": "test"}, id="invalid base-image"), + pytest.param( + {"--callback-script": "non-existant-path"}, id="empty image name positional argument" + ), pytest.param({"": ""}, id="empty cloud name positional argument"), pytest.param({" ": ""}, id="empty image name positional argument"), ], ) -def test_main_invalid_run_inputs( - monkeypatch: pytest.MonkeyPatch, - run_inputs: dict[str, str], - invalid_patch: dict[str, str], -): +def test__parse_args_invalid_run_args(run_inputs: dict, invalid_args: dict): """ - arrange: given invalid run arguments and mocked builder functions. - act: when main is called. - assert: SystemExit is raised. + arrange: given invalid run action arguments. + act: when _parse_args is called. + assert: SystemExit error is raised. """ - monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) - monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) - monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - run_inputs.update(invalid_patch) + run_inputs.update(invalid_args) inputs = list( # if flag does not exist, append it as a positional argument. itertools.chain.from_iterable( @@ -166,42 +149,164 @@ def test_main_invalid_run_inputs( ) ) - with pytest.raises(SystemExit): - main(["run", *inputs]) + with pytest.raises(SystemExit) as exc: + cli._parse_args(inputs) - initialize_mock.assert_not_called() - get_mock.assert_not_called() - build_mock.assert_not_called() + assert "invalid choice" in str(exc.getrepr()) @pytest.mark.parametrize( - "image", + "action, args, expected", [ - pytest.param("jammy", id="jammy"), - pytest.param("22.04", id="jammy tag"), - pytest.param("noble", id="noble"), - pytest.param("24.04", id="noble tag"), + pytest.param("init", {}, cli.argparse.Namespace(action="init"), id="init"), + pytest.param( + "latest-build-id", + {"": "test-cloud-name", " ": "test-image-name"}, + cli.argparse.Namespace( + action="latest-build-id", + cloud_name="test-cloud-name", + image_name="test-image-name", + ), + id="latest-build-id", + ), + pytest.param( + "run", + {"": "test-cloud-name", " ": "test-image-name"}, + cli.argparse.Namespace( + action="run", + base="noble", + callback_script_path=None, + cloud_name="test-cloud-name", + image_name="test-image-name", + keep_revisions=5, + ), + id="run (no-optional)", + ), + pytest.param( + "run", + {"": "test-cloud-name", " ": "test-image-name", "--base-image": "jammy"}, + cli.argparse.Namespace( + action="run", + base="jammy", + callback_script_path=None, + cloud_name="test-cloud-name", + image_name="test-image-name", + keep_revisions=5, + ), + id="run (base image)", + ), + pytest.param( + "run", + {"": "test-cloud-name", " ": "test-image-name", "--keep-revisions": "2"}, + cli.argparse.Namespace( + action="run", + base="noble", + callback_script_path=None, + cloud_name="test-cloud-name", + image_name="test-image-name", + keep_revisions=2, + ), + id="run (keep revisions)", + ), + pytest.param( + "run", + {"": "test-cloud-name", " ": "test-image-name", "--callback-script": "test_callback"}, + cli.argparse.Namespace( + action="run", + base="noble", + callback_script_path=Path("test_callback"), + cloud_name="test-cloud-name", + image_name="test-image-name", + keep_revisions=5, + ), + id="run (callback script)", + ), ], ) -def test_main_run(monkeypatch: pytest.MonkeyPatch, image: str, run_inputs: dict[str, str]): +def test__parse_args( + monkeypatch: pytest.MonkeyPatch, action: str, args: dict, expected: cli.ActionsNamespace +): """ - arrange: given invalid run argument and mocked builder functions. - act: when main is called. - assert: run is called. + arrange: given action and its arguments. + act: when _parse_args is called. + assert: expected ActionsNamespace object is created. """ - monkeypatch.setattr(cli.builder, "initialize", (initialize_mock := MagicMock())) - monkeypatch.setattr(cli.store, "get_latest_build_id", (get_mock := MagicMock())) - monkeypatch.setattr(cli, "_build_and_upload", (build_mock := MagicMock())) - run_inputs.update({"--base-image": image}) + monkeypatch.setattr( + cli, + "_existing_path", + MagicMock( + return_value=( + Path(args["--callback-script"]) if args.get("--callback-script", None) else None + ) + ), + ) inputs = list( # if flag does not exist, append it as a positional argument. itertools.chain.from_iterable( - (flag, value) if flag.strip() else (value,) for (flag, value) in run_inputs.items() + (flag, value) if flag.strip() else (value,) for (flag, value) in args.items() ) ) - main(["run", *inputs]) + assert cli._parse_args([action, *inputs]) == expected + + +def test__existing_path_not_exists(tmp_path: Path): + """ + arrange: given a path that does not exist. + act: when _existing_path is called. + assert: ValueError is raised. + """ + not_exists_path = tmp_path / "non-existent" + with pytest.raises(ValueError) as exc: + cli._existing_path(str(not_exists_path)) + + assert f"Given path {not_exists_path} not found." in str(exc.getrepr()) + + +def test__existing_path(tmp_path: Path): + """ + arrange: given a path that does not exist. + act: when _existing_path is called. + assert: ValueError is raised. + """ + not_exists_path = tmp_path / "non-existent" + not_exists_path.touch() + assert cli._existing_path(str(not_exists_path)) == not_exists_path + + +def test__non_empty_string_error(): + """ + arrange: given an empty string. + act: when _non_empty_string is called. + assert: ValueError is raised. + """ + with pytest.raises(ValueError): + cli._non_empty_string("") + - initialize_mock.assert_not_called() - get_mock.assert_not_called() - build_mock.assert_called() +@pytest.mark.parametrize( + "callback_script", + [ + pytest.param(None, id="No callback script"), + pytest.param(Path("tmp_path"), id="Callback script"), + ], +) +def test__build_and_upload(monkeypatch: pytest.MonkeyPatch, callback_script: Path | None): + """ + arrange: given a monkeypatched builder.setup_builder function. + act: when _build is called. + assert: the mock function is called. + """ + monkeypatch.setattr(cli.builder, "build_image", (builder_mock := MagicMock())) + monkeypatch.setattr(cli.store, "upload_image", MagicMock(return_value="test-image-id")) + monkeypatch.setattr(cli.subprocess, "check_call", MagicMock()) + + cli._build_and_upload( + base="jammy", + cloud_name=MagicMock(), + image_name=MagicMock(), + keep_revisions=MagicMock(), + callback_script_path=callback_script, + ) + + builder_mock.assert_called_once() From 759ddd89d78b90ae4af039ab758f70bc0a1ba2b0 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 04:27:22 +0000 Subject: [PATCH 26/63] add checksum checking --- src/github_runner_image_builder/builder.py | 108 ++++++- test.txt | 1 + tests/unit/test_builder.py | 348 +++++++++++++++------ 3 files changed, 346 insertions(+), 111 deletions(-) create mode 100644 test.txt diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index f3debfb..8e05750 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -3,6 +3,7 @@ """Module for interacting with qemu image builder.""" +import hashlib import logging import shutil @@ -13,6 +14,8 @@ from pathlib import Path from typing import Literal +import requests + from github_runner_image_builder.chroot import ChrootBaseError, ChrootContextManager from github_runner_image_builder.config import IMAGE_OUTPUT_PATH, Arch, BaseImage from github_runner_image_builder.errors import ( @@ -167,7 +170,7 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: logger.info("Cleaning build state.") _clean_build_state() logger.info("Downloading base image.") - base_image_path = _download_base_image(arch=arch, base_image=base_image) + base_image_path = _download_and_validate_image(arch=arch, base_image=base_image) logger.info("Resizing base image.") _resize_image(image_path=base_image_path) logger.info("Mounting network block device.") @@ -280,8 +283,8 @@ def _clean_build_state() -> None: raise CleanBuildStateError from exc -def _download_base_image(arch: Arch, base_image: BaseImage) -> Path: - """Download the base image from cloud-images.ubuntu.com. +def _download_and_validate_image(arch: Arch, base_image: BaseImage) -> Path: + """Download and verify the base image from cloud-images.ubuntu.com. Args: arch: The base image architecture to download. @@ -291,26 +294,23 @@ def _download_base_image(arch: Arch, base_image: BaseImage) -> Path: The downloaded image path. Raises: - BaseImageDownloadError: If there was an error downloading the image. + BaseImageDownloadError: If there was an error with downloading/verifying the image. """ try: bin_arch = _get_supported_runner_arch(arch) except UnsupportedArchitectureError as exc: raise BaseImageDownloadError from exc - try: - # The ubuntu-cloud-images is a trusted source - image_path = f"{base_image.value}-server-cloudimg-{bin_arch}.img" - urllib.request.urlretrieve( # nosec: B310 - ( - f"https://cloud-images.ubuntu.com/{base_image.value}/current/{base_image.value}" - f"-server-cloudimg-{bin_arch}.img" - ), - image_path, - ) - return Path(image_path) - except urllib.error.URLError as exc: - raise BaseImageDownloadError from exc + image_path_str = f"{base_image.value}-server-cloudimg-{bin_arch}.img" + image_path = _download_base_image( + base_image=base_image, bin_arch=bin_arch, output_filename=image_path_str + ) + shasums = _fetch_shasums(base_image=base_image) + if image_path_str not in shasums: + raise BaseImageDownloadError("Corresponding checksum not found.") + if not _validate_checksum(image_path, shasums[image_path_str]): + raise BaseImageDownloadError("Invalid checksum.") + return image_path def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: @@ -340,6 +340,80 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") +def _download_base_image(base_image: BaseImage, bin_arch: str, output_filename: str) -> Path: + """Download the base image. + + Args: + bin_arch: The ubuntu cloud-image supported arch. + base_image: The ubuntu base image OS to download. + output_filename: The output filename of the downloaded image. + + Raises: + BaseImageDownloadError: If there was an error downloaded from cloud-images.ubuntu.com + + Returns: + The downloaded image path. + """ + # The ubuntu-cloud-images is a trusted source + try: + urllib.request.urlretrieve( # nosec: B310 + ( + f"https://cloud-images.ubuntu.com/{base_image.value}/current/{base_image.value}" + f"-server-cloudimg-{bin_arch}.img" + ), + output_filename, + ) + except urllib.error.URLError as exc: + raise BaseImageDownloadError from exc + return Path(output_filename) + + +def _fetch_shasums(base_image: BaseImage) -> dict[str, str]: + """Fetch SHA256SUM for given base image. + + Args: + base_image: The ubuntu base image OS to fetch SHA256SUMs for. + + Raises: + BaseImageDownloadError: If there was an error downloading SHA256SUMS file from \ + cloud-images.ubuntu.com + + Returns: + A map of image file name to SHA256SUM. + """ + try: + # bandit does not detect that the timeout parameter exists. + response = requests.get( # nosec: request_without_timeout + f"https://cloud-images.ubuntu.com/{base_image.value}/current/SHA256SUMS", + timeout=60 * 5, + ) + except requests.RequestException as exc: + raise BaseImageDownloadError from exc + # file consisting of lines * + shasum_contents = str(response.content, encoding="utf-8") + imagefile_to_shasum = { + sha256_and_file[1].strip("*"): sha256_and_file[0] + for shasum_line in shasum_contents.strip().splitlines() + if (sha256_and_file := shasum_line.split()) + } + return imagefile_to_shasum + + +def _validate_checksum(file: Path, expected_checksum: str) -> bool: + """Validate the checksum of a given file. + + Args: + file: The file to calculate checksum for. + expected_checksum: The expected file checksum. + + Returns: + True if the checksums match. False otherwise. + """ + sha256 = hashlib.sha256() + sha256.update(file.read_bytes()) + return sha256.hexdigest() == expected_checksum + + def _resize_image(image_path: Path) -> None: """Resize image to allow space for dependency installations. diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..d950c5d --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +sha256sumteststring diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 3489ff1..b4ddba1 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -122,31 +122,64 @@ def test__enable_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): assert "Module nbd not found" in str(exc.getrepr()) -def test__get_supported_runner_arch_unsupported_error(): - """ - arrange: given an architecture value that isn't supported. - act: when _get_supported_runner_arch is called. - assert: UnsupportedArchitectureError is raised. - """ - arch = MagicMock() - with pytest.raises(UnsupportedArchitectureError): - builder._get_supported_runner_arch(arch) - - @pytest.mark.parametrize( - "arch, expected", + "patch_obj, sub_func, mock, expected_message", [ - pytest.param(Arch.ARM64, "arm64", id="ARM64"), - pytest.param(Arch.X64, "amd64", id="AMD64"), + pytest.param( + builder, + "_resize_mount_partitions", + MagicMock(side_effect=ResizePartitionError("Partition resize failed")), + "Partition resize failed", + id="Partition resize failed", + ), + pytest.param( + builder, + "ChrootContextManager", + MagicMock(side_effect=ChrootBaseError("Failed to chroot into dir")), + "Failed to chroot into dir", + id="Failed to chroot into dir", + ), + pytest.param( + builder, + "_compress_image", + MagicMock(side_effect=ImageCompressError("Failed to compress image")), + "Failed to compress image", + id="Failed to compress image", + ), ], ) -def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch): +def test_build_image_error( + patch_obj: Any, + sub_func: str, + mock: MagicMock, + expected_message: str, + monkeypatch: pytest.MonkeyPatch, +): """ - arrange: given an architecture value that is supported. - act: when _get_supported_runner_arch is called. - assert: Expected architecture in cloud_images format is returned. + arrange: given a monkeypatched functions of build_image that raises exceptions. + act: when build_image is called. + assert: BuildImageError is raised. """ - assert builder._get_supported_runner_arch(arch) == expected + monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", MagicMock()) + monkeypatch.setattr(builder, "_clean_build_state", MagicMock()) + monkeypatch.setattr(builder, "_download_and_validate_image", MagicMock()) + monkeypatch.setattr(builder, "_resize_image", MagicMock()) + monkeypatch.setattr(builder, "_mount_image_to_network_block_device", MagicMock()) + monkeypatch.setattr(builder, "_resize_mount_partitions", MagicMock()) + monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) + monkeypatch.setattr(builder, "_install_yq", MagicMock()) + monkeypatch.setattr(builder, "ChrootContextManager", MagicMock()) + monkeypatch.setattr(builder.subprocess, "check_output", MagicMock()) + monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) + monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) + monkeypatch.setattr(builder, "_install_yarn", MagicMock()) + monkeypatch.setattr(builder, "_compress_image", MagicMock()) + monkeypatch.setattr(patch_obj, sub_func, mock) + + with pytest.raises(BuildImageError) as exc: + builder.build_image(arch=MagicMock(), base_image=MagicMock()) + + assert expected_message in str(exc.getrepr()) def test__clean_build_state_error(monkeypatch: pytest.MonkeyPatch): @@ -193,15 +226,22 @@ def test__clean_build_state(monkeypatch: pytest.MonkeyPatch): id="Unsupported architecture", ), pytest.param( - builder.urllib.request, - "urlretrieve", - builder.urllib.error.ContentTooShortError("Network interrupted", ""), - "Network interrupted", + builder, + "_download_base_image", + BaseImageDownloadError("Content too short"), + "Content too short", id="Network interrupted", ), + pytest.param( + builder, + "_fetch_shasums", + BaseImageDownloadError("Content too short"), + "Content too short", + id="Network interrupted (SHASUM)", + ), ], ) -def test__download_base_image_fail( +def test__download_and_validate_image_error( patch_obj: Any, sub_func: str, exception: Exception, @@ -209,35 +249,215 @@ def test__download_base_image_fail( monkeypatch: pytest.MonkeyPatch, ): """ - arrange: given monkeypatched sub functions of _download_base_image that raises exceptions. - act: when _download_base_image is called. - assert: A CloudImageDownloadError is raised. + arrange: given monkeypatched sub functions of _download_and_validate_image that raises \ + exceptions. + act: when _download_and_validate_image is called. + assert: A BaseImageDownloadError is raised. """ mock_func = MagicMock(side_effect=exception) monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock) - monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock) + monkeypatch.setattr(builder, "_download_base_image", MagicMock) + monkeypatch.setattr(builder, "_fetch_shasums", MagicMock) + monkeypatch.setattr(builder, "_validate_checksum", MagicMock) monkeypatch.setattr(patch_obj, sub_func, mock_func) with pytest.raises(BaseImageDownloadError) as exc: - builder._download_base_image(arch=MagicMock(), base_image=MagicMock()) + builder._download_and_validate_image(arch=MagicMock(), base_image=MagicMock()) assert expected_message in str(exc.getrepr()) +def test__download_and_validate_image_no_shasum( + monkeypatch: pytest.MonkeyPatch, +): + """ + arrange: given monkeypatched _fetch_shasums that returns empty shasums. + act: when _download_and_validate_image is called. + assert: A BaseImageDownloadError is raised. + """ + monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock()) + monkeypatch.setattr(builder, "_download_base_image", MagicMock()) + monkeypatch.setattr(builder, "_fetch_shasums", MagicMock(return_value={})) + + with pytest.raises(BaseImageDownloadError) as exc: + builder._download_and_validate_image(arch=MagicMock(), base_image=MagicMock()) + + assert "Corresponding checksum not found." in str(exc.getrepr()) + + +def test__download_and_validate_image_invalid_checksum( + monkeypatch: pytest.MonkeyPatch, +): + """ + arrange: given monkeypatched _validate_checksum that returns false. + act: when _download_and_validate_image is called. + assert: A BaseImageDownloadError is raised. + """ + monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock(return_value="x64")) + monkeypatch.setattr(builder, "_download_base_image", MagicMock()) + monkeypatch.setattr( + builder, + "_fetch_shasums", + MagicMock(return_value={"jammy-server-cloudimg-x64.img": "test"}), + ) + monkeypatch.setattr(builder, "_validate_checksum", MagicMock(return_value=False)) + + with pytest.raises(BaseImageDownloadError) as exc: + builder._download_and_validate_image(arch=Arch.X64, base_image=BaseImage.JAMMY) + + assert "Invalid checksum." in str(exc.getrepr()) + + +def test__download_and_validate_image(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given monkeypatched sub functions of _download_and_validate_image. + act: when _download_and_validate_image is called. + assert: the mocked subfunctions are called. + """ + monkeypatch.setattr( + builder, "_get_supported_runner_arch", get_arch_mock := MagicMock(return_value="x64") + ) + monkeypatch.setattr(builder, "_download_base_image", download_base_mock := MagicMock()) + monkeypatch.setattr( + builder, + "_fetch_shasums", + fetch_shasums_mock := MagicMock(return_value={"jammy-server-cloudimg-x64.img": "test"}), + ) + monkeypatch.setattr(builder, "_validate_checksum", validate_checksum_mock := MagicMock()) + + builder._download_and_validate_image(arch=Arch.X64, base_image=BaseImage.JAMMY) + + get_arch_mock.assert_called_once() + download_base_mock.assert_called_once() + fetch_shasums_mock.assert_called_once() + validate_checksum_mock.assert_called_once() + + +def test__get_supported_runner_arch_unsupported_error(): + """ + arrange: given an architecture value that isn't supported. + act: when _get_supported_runner_arch is called. + assert: UnsupportedArchitectureError is raised. + """ + arch = MagicMock() + with pytest.raises(UnsupportedArchitectureError): + builder._get_supported_runner_arch(arch) + + +@pytest.mark.parametrize( + "arch, expected", + [ + pytest.param(Arch.ARM64, "arm64", id="ARM64"), + pytest.param(Arch.X64, "amd64", id="AMD64"), + ], +) +def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch): + """ + arrange: given an architecture value that is supported. + act: when _get_supported_runner_arch is called. + assert: Expected architecture in cloud_images format is returned. + """ + assert builder._get_supported_runner_arch(arch) == expected + + +def test__download_base_image_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given monkeypatched urlretrieve function that raises an error. + act: when _download_base_image is called. + assert: BaseImageDownloadError is raised. + """ + monkeypatch.setattr( + builder.urllib.request, + "urlretrieve", + MagicMock(side_effect=builder.urllib.error.URLError(reason="Content too short")), + ) + + with pytest.raises(BaseImageDownloadError) as exc: + builder._download_base_image( + base_image=MagicMock(), bin_arch=MagicMock(), output_filename=MagicMock() + ) + + assert "Content too short" in str(exc.getrepr()) + + def test__download_base_image(monkeypatch: pytest.MonkeyPatch): """ - arrange: given monkeypatched sub functions of _download_base_image. + arrange: given monkeypatched urlretrieve function. act: when _download_base_image is called. - assert: the downloaded path is returned. + assert: Path from output_filename input is returned. """ - monkeypatch.setattr(builder, "_get_supported_runner_arch", MagicMock(return_value="amd64")) monkeypatch.setattr(builder.urllib.request, "urlretrieve", MagicMock()) + test_file_name = "test_file_name" - assert builder._download_base_image(arch=Arch.X64, base_image=BaseImage.JAMMY) == Path( - "jammy-server-cloudimg-amd64.img" + assert Path("test_file_name") == builder._download_base_image( + base_image=MagicMock(), bin_arch=MagicMock(), output_filename=test_file_name ) +def test__fetch_shasums_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given monkeypatched requests function that raises an error. + act: when _fetch_shasums is called. + assert: BaseImageDownloadError is raised. + """ + monkeypatch.setattr( + builder.requests, + "get", + MagicMock(side_effect=builder.requests.RequestException("Content too short")), + ) + + with pytest.raises(BaseImageDownloadError) as exc: + builder._fetch_shasums(base_image=MagicMock()) + + assert "Content too short" in str(exc.getrepr()) + + +def test__fetch_shasums(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given monkeypatched requests function that returns mocked contents of SHA256SUMS. + act: when _fetch_shasums is called. + assert: a dictionary with filename to shasum is created. + """ + mock_response = MagicMock() + mock_response.content = bytes( + """test_shasum1 *file1 +test_shasum2 *file2 +test_shasum3 *file3 +""", + encoding="utf-8", + ) + monkeypatch.setattr(builder.requests, "get", MagicMock(return_value=mock_response)) + + assert { + "file1": "test_shasum1", + "file2": "test_shasum2", + "file3": "test_shasum3", + } == builder._fetch_shasums(base_image=MagicMock()) + + +@pytest.mark.parametrize( + "content, checksum, expected", + [ + pytest.param( + "sha256sumteststring", + "52b60ec50ea69cd09d5f25b75c295b93181eaba18444fdbc537beee4653bad7e", + True, + ), + pytest.param("test", "test", False), + ], +) +def test__validate_checksum(tmp_path: Path, content: str, checksum: str, expected: bool): + """ + arrange: given a file content and a checksum pair. + act: when _validate_checksum is called. + assert: expected result is returned. + """ + test_path = tmp_path / "test" + test_path.write_text(content, encoding="utf-8") + + assert expected == builder._validate_checksum(test_path, checksum) + + def test__resize_image_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched subprocess.run that raises an exception. @@ -516,63 +736,3 @@ def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): builder._compress_image(image=MagicMock()) assert "Compression error" in str(exc.getrepr()) - - -@pytest.mark.parametrize( - "patch_obj, sub_func, mock, expected_message", - [ - pytest.param( - builder, - "_resize_mount_partitions", - MagicMock(side_effect=ResizePartitionError("Partition resize failed")), - "Partition resize failed", - id="Partition resize failed", - ), - pytest.param( - builder, - "ChrootContextManager", - MagicMock(side_effect=ChrootBaseError("Failed to chroot into dir")), - "Failed to chroot into dir", - id="Failed to chroot into dir", - ), - pytest.param( - builder, - "_compress_image", - MagicMock(side_effect=ImageCompressError("Failed to compress image")), - "Failed to compress image", - id="Failed to compress image", - ), - ], -) -def test_build_image_error( - patch_obj: Any, - sub_func: str, - mock: MagicMock, - expected_message: str, - monkeypatch: pytest.MonkeyPatch, -): - """ - arrange: given a monkeypatched functions of build_image that raises exceptions. - act: when build_image is called. - assert: BuildImageError is raised. - """ - monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", MagicMock()) - monkeypatch.setattr(builder, "_clean_build_state", MagicMock()) - monkeypatch.setattr(builder, "_download_base_image", MagicMock()) - monkeypatch.setattr(builder, "_resize_image", MagicMock()) - monkeypatch.setattr(builder, "_mount_image_to_network_block_device", MagicMock()) - monkeypatch.setattr(builder, "_resize_mount_partitions", MagicMock()) - monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) - monkeypatch.setattr(builder, "_install_yq", MagicMock()) - monkeypatch.setattr(builder, "ChrootContextManager", MagicMock()) - monkeypatch.setattr(builder.subprocess, "check_output", MagicMock()) - monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) - monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) - monkeypatch.setattr(builder, "_install_yarn", MagicMock()) - monkeypatch.setattr(builder, "_compress_image", MagicMock()) - monkeypatch.setattr(patch_obj, sub_func, mock) - - with pytest.raises(BuildImageError) as exc: - builder.build_image(arch=MagicMock(), base_image=MagicMock()) - - assert expected_message in str(exc.getrepr()) From 9cd60f6b0f3542f9547863ec9d074481619a8b08 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 05:02:18 +0000 Subject: [PATCH 27/63] delete test file --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index d950c5d..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -sha256sumteststring From 504829a0536ef7c00d904cee2dedb386fa2aec4c Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 05:02:39 +0000 Subject: [PATCH 28/63] execute callback script --- src/github_runner_image_builder/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index f8425a1..6562868 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -205,4 +205,4 @@ def _build_and_upload( ) if callback_script_path: # The callback script is a user trusted script. - subprocess.check_call([str(callback_script_path), image_id]) # nosec: B603 + subprocess.check_call([f"./{callback_script_path}", image_id]) # nosec: B603 From e55f9ab32df246fbe3acc357c2ce98160e8eb9d0 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 07:15:27 +0000 Subject: [PATCH 29/63] checksum validation bytes --- src/github_runner_image_builder/builder.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 8e05750..b0552d7 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -50,6 +50,8 @@ APT_NONINTERACTIVE_ENV = {"DEBIAN_FRONTEND": "noninteractive"} SNAP_GO = "go" +CHECKSUM_BUF_SIZE = 65536 # 65kb + # Constants for mounting images IMAGE_MOUNT_DIR = Path("/mnt/ubuntu-image/") NETWORK_BLOCK_DEVICE_PATH = Path("/dev/nbd0") @@ -410,7 +412,12 @@ def _validate_checksum(file: Path, expected_checksum: str) -> bool: True if the checksums match. False otherwise. """ sha256 = hashlib.sha256() - sha256.update(file.read_bytes()) + with open(file=file, mode="rb") as target_file: + while True: + data = target_file.read(CHECKSUM_BUF_SIZE) + if not data: + break + sha256.update(data) return sha256.hexdigest() == expected_checksum From b1576cd8fc43ac908f5d258e513b484e41dad42d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 08:18:47 +0000 Subject: [PATCH 30/63] add defensive exceptions for subprocess errors --- src/github_runner_image_builder/builder.py | 18 +++++ tests/unit/test_builder.py | 78 +++++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index b0552d7..62af9f9 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -444,6 +444,8 @@ def _resize_image(image_path: Path) -> None: exc.output, ) raise ImageResizeError from exc + except subprocess.SubprocessError as exc: + raise ImageResizeError from exc def _mount_image_to_network_block_device(image_path: Path) -> None: @@ -470,6 +472,8 @@ def _mount_image_to_network_block_device(image_path: Path) -> None: exc.output, ) raise ImageMountError from exc + except subprocess.SubprocessError as exc: + raise ImageMountError from exc # Network block device may fail to mount, retrying will usually fix this. @@ -513,6 +517,8 @@ def _resize_mount_partitions() -> None: exc.output, ) raise ResizePartitionError from exc + except subprocess.SubprocessError as exc: + raise ResizePartitionError from exc def _install_yq() -> None: @@ -548,6 +554,8 @@ def _install_yq() -> None: exc.output, ) raise YQBuildError from exc + except subprocess.SubprocessError as exc: + raise YQBuildError from exc def _replace_mounted_resolv_conf() -> None: @@ -608,6 +616,8 @@ def _disable_unattended_upgrades() -> None: exc.output, ) raise UnattendedUpgradeDisableError from exc + except subprocess.SubprocessError as exc: + raise UnattendedUpgradeDisableError from exc def _configure_system_users() -> None: @@ -647,6 +657,8 @@ def _configure_system_users() -> None: exc.output, ) raise SystemUserConfigurationError from exc + except subprocess.SubprocessError as exc: + raise SystemUserConfigurationError from exc def _configure_usr_local_bin() -> None: @@ -669,6 +681,8 @@ def _configure_usr_local_bin() -> None: exc.output, ) raise PermissionConfigurationError from exc + except subprocess.SubprocessError as exc: + raise PermissionConfigurationError from exc def _install_yarn() -> None: @@ -695,6 +709,8 @@ def _install_yarn() -> None: exc.output, ) raise YarnInstallError from exc + except subprocess.SubprocessError as exc: + raise YarnInstallError from exc # Image compression might fail for arbitrary reasons - retrying usually solves this. @@ -722,3 +738,5 @@ def _compress_image(image: Path) -> None: exc.output, ) raise ImageCompressError from exc + except subprocess.SubprocessError as exc: + raise ImageCompressError from exc diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index b4ddba1..8157a59 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -8,7 +8,7 @@ import time from pathlib import Path -from typing import Any +from typing import Any, Type from unittest.mock import MagicMock import pytest @@ -57,7 +57,7 @@ def test_subprocess_call_funcs( ): """ arrange: given functions that consist of subprocess calls only with mocked subprocess calls. - act: when the functions are called. + act: when the function is called. assert: no errors are raised. """ monkeypatch.setattr(subprocess, "check_output", MagicMock()) @@ -69,6 +69,80 @@ def test_subprocess_call_funcs( assert getattr(builder, func)(*args) is None +@pytest.mark.parametrize( + "func, args, exc", + [ + pytest.param( + "_clean_build_state", [], builder.CleanBuildStateError, id="clean build state" + ), + pytest.param("_resize_image", [MagicMock()], builder.ImageResizeError, id="resize image"), + pytest.param( + "_mount_image_to_network_block_device", + [MagicMock()], + builder.ImageMountError, + id="mount image to nbd", + ), + pytest.param( + "_resize_mount_partitions", [], builder.ResizePartitionError, id="resize mount parts" + ), + pytest.param("_install_yq", [], builder.YQBuildError, id="install yq"), + pytest.param( + "_disable_unattended_upgrades", + [], + builder.UnattendedUpgradeDisableError, + id="disable unattende upgrades", + ), + pytest.param( + "_configure_system_users", + [], + builder.SystemUserConfigurationError, + id="configure system users", + ), + pytest.param( + "_configure_usr_local_bin", + [], + builder.PermissionConfigurationError, + id="configure system users", + ), + pytest.param( + "_install_yarn", + [], + builder.YarnInstallError, + id="install yarn", + ), + pytest.param( + "_compress_image", + [MagicMock()], + builder.ImageCompressError, + id="compress image", + ), + ], +) +def test_subprocess_func_errors( + monkeypatch: pytest.MonkeyPatch, func: str, args: list[Any], exc: Type[Exception] +): + """ + arrange: given functions with subprocess calls that is monkeypatched to raise exceptions. + act: when the function is called. + assert: subprocess error is wrapped to expected error. + """ + monkeypatch.setattr( + subprocess, + "check_output", + MagicMock(side_effect=subprocess.SubprocessError("Test subprocess error")), + ) + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=subprocess.SubprocessError("Test subprocess error")), + ) + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) + + with pytest.raises(exc): + getattr(builder, func)(*args) + + def test_initialize(monkeypatch: pytest.MonkeyPatch): """ arrange: given sub functions of initialize. From 8e1a47838a5719238b6b829477597c7012b582f5 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 08:18:56 +0000 Subject: [PATCH 31/63] raise on delete fail --- src/github_runner_image_builder/store.py | 8 +++++--- tests/unit/test_store.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/github_runner_image_builder/store.py b/src/github_runner_image_builder/store.py index 7cbaed5..e7bd7b8 100644 --- a/src/github_runner_image_builder/store.py +++ b/src/github_runner_image_builder/store.py @@ -57,6 +57,9 @@ def _prune_old_images( connection: The connected openstack cloud instance. image_name: The image name to search for. num_revisions: The number of revisions to keep. + + Raises: + OpenstackError: if there was an error deleting the images. """ images = _get_sorted_images_by_created_at(connection=connection, image_name=image_name) if not images: @@ -65,10 +68,9 @@ def _prune_old_images( for image in images_to_prune: try: if not connection.delete_image(image.id, wait=True): - logger.error("Failed to delete old image, %s", image.id) + raise OpenstackError(f"Failed to delete image: {image.id}") except openstack.exceptions.OpenStackCloudException as exc: - logger.error("Failed to prune old image, %s", exc) - continue + raise OpenstackError from exc def get_latest_build_id(cloud_name: str, image_name: str) -> str | None: diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index fa044c4..8c5e230 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -60,7 +60,7 @@ def test__get_sorted_images_by_created_at(mock_connection: MagicMock): ) == [third, second, first] -def test__prune_old_images_error(caplog: pytest.LogCaptureFixture, mock_connection: MagicMock): +def test__prune_old_images_error(mock_connection: MagicMock): """ arrange: given a mocked delete function that raises an exception. act: when _prune_old_images is called. @@ -74,12 +74,13 @@ def test__prune_old_images_error(caplog: pytest.LogCaptureFixture, mock_connecti "Delete error" ) - store._prune_old_images(connection=mock_connection, image_name=MagicMock(), num_revisions=0) - - assert all("Failed to prune old image" in log for log in caplog.messages) + with pytest.raises(OpenstackError): + store._prune_old_images( + connection=mock_connection, image_name=MagicMock(), num_revisions=0 + ) -def test__prune_old_images_fail(caplog: pytest.LogCaptureFixture, mock_connection: MagicMock): +def test__prune_old_images_fail(mock_connection: MagicMock): """ arrange: given a mocked delete function that returns false. act: when _prune_old_images is called. @@ -91,9 +92,10 @@ def test__prune_old_images_fail(caplog: pytest.LogCaptureFixture, mock_connectio ] mock_connection.delete_image.return_value = False - store._prune_old_images(connection=mock_connection, image_name=MagicMock(), num_revisions=0) - - assert all("Failed to delete old image" in log for log in caplog.messages) + with pytest.raises(OpenstackError): + store._prune_old_images( + connection=mock_connection, image_name=MagicMock(), num_revisions=0 + ) def test__prune_old_images(mock_connection: MagicMock): From 4b57df4de2a938318f71af00e490e59b9ed9b26b Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 10:04:45 +0000 Subject: [PATCH 32/63] test disconnect cleanup --- src/github_runner_image_builder/builder.py | 126 ++++++++------------- src/github_runner_image_builder/chroot.py | 1 + src/github_runner_image_builder/errors.py | 8 +- tests/unit/test_builder.py | 10 +- 4 files changed, 54 insertions(+), 91 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 62af9f9..2f5786a 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -21,10 +21,9 @@ from github_runner_image_builder.errors import ( BaseImageDownloadError, BuildImageError, - CleanBuildStateError, DependencyInstallError, ImageCompressError, - ImageMountError, + ImageConnectError, ImageResizeError, NetworkBlockDeviceError, PermissionConfigurationError, @@ -169,14 +168,12 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: BuildImageError: If there was an error building the image. """ IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) - logger.info("Cleaning build state.") - _clean_build_state() logger.info("Downloading base image.") base_image_path = _download_and_validate_image(arch=arch, base_image=base_image) logger.info("Resizing base image.") _resize_image(image_path=base_image_path) - logger.info("Mounting network block device.") - _mount_image_to_network_block_device(image_path=base_image_path) + logger.info("Connecting image to network block device.") + _connect_image_to_network_block_device(image_path=base_image_path) logger.info("Resizing partitions.") _resize_mount_partitions() logger.info("Installing YQ from source.") @@ -213,78 +210,14 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: _install_yarn() except ChrootBaseError as exc: raise BuildImageError from exc - finally: - _clean_build_state() + + logger.info("Disconnecting image to network block device.") + _disconnect_image_to_network_block_device() logger.info("Compressing image.") _compress_image(base_image_path) -def _clean_build_state() -> None: - """Remove any artefacts left by previous build. - - Raises: - CleanBuildStateError: if there was an error cleaning up the build state. - """ - # The commands will fail if artefacts do not exist and hence there is no need to check the - # output of subprocess runs. - try: - output = subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], - timeout=30, - check=False, - ) # nosec: B603 - logger.info("umount dev out: %s", output) - output = subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], - timeout=30, - check=False, - capture_output=True, - ) # nosec: B603 - logger.info("umount proc out: %s", output) - output = subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], - timeout=30, - check=False, - capture_output=True, - ) # nosec: B603 - logger.info("umount sys out: %s", output) - output = subprocess.run( - ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False, capture_output=True - ) # nosec: B603 - logger.info("umount ubuntu-image out: %s", output) - output = subprocess.run( - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], - timeout=30, - check=False, - capture_output=True, - ) # nosec: B603 - logger.info("umount nbd out: %s", output) - output = subprocess.run( # nosec: B603 - ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], - timeout=30, - check=False, - capture_output=True, - ) - logger.info("umount nbdp1 out: %s", output) - output = subprocess.run( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], - timeout=30, - check=False, - capture_output=True, - ) - logger.info("qemu-nbd disconnect nbd out: %s", output) - output = subprocess.run( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], - timeout=30, - check=False, - capture_output=True, - ) - logger.info("qemu-nbd disconnect nbdp1 out: %s", output) - except subprocess.SubprocessError as exc: - raise CleanBuildStateError from exc - - def _download_and_validate_image(arch: Arch, base_image: BaseImage) -> Path: """Download and verify the base image from cloud-images.ubuntu.com. @@ -448,14 +381,14 @@ def _resize_image(image_path: Path) -> None: raise ImageResizeError from exc -def _mount_image_to_network_block_device(image_path: Path) -> None: - """Mount the image to network block device in preparation for chroot. +def _connect_image_to_network_block_device(image_path: Path) -> None: + """Connect the image to network block device in preparation for chroot. Args: - image_path: The target image file to mount. + image_path: The target image file to connect. Raises: - ImageMountError: If there was an error mounting the image to network block device. + ImageConnectError: If there was an error connecting the image to network block device. """ try: output = subprocess.check_output( # nosec: B603 @@ -466,14 +399,14 @@ def _mount_image_to_network_block_device(image_path: Path) -> None: _mount_network_block_device_partition() except subprocess.CalledProcessError as exc: logger.exception( - "Error mounting image to network block device, cmd: %s, code: %s, err: %s", + "Error connecting image to network block device, cmd: %s, code: %s, err: %s", exc.cmd, exc.returncode, exc.output, ) - raise ImageMountError from exc + raise ImageConnectError from exc except subprocess.SubprocessError as exc: - raise ImageMountError from exc + raise ImageConnectError from exc # Network block device may fail to mount, retrying will usually fix this. @@ -713,6 +646,39 @@ def _install_yarn() -> None: raise YarnInstallError from exc +def _disconnect_image_to_network_block_device(): + """Disconnect the image to network block device in cleanup for chroot. + + Raises: + ImageConnectError: If there was an error disconnecting the image from network block device. + """ + try: + output = subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], + timeout=30, + check=True, + capture_output=True, + ) + logger.info("qemu-nbd disconnect nbd out: %s", output) + output = subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + timeout=30, + check=True, + capture_output=True, + ) + logger.info("qemu-nbd disconnect nbdp1 out: %s", output) + except subprocess.CalledProcessError as exc: + logger.exception( + "Error disconnecting image to network block device, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) + raise ImageConnectError from exc + except subprocess.SubprocessError as exc: + raise ImageConnectError from exc + + # Image compression might fail for arbitrary reasons - retrying usually solves this. @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _compress_image(image: Path) -> None: diff --git a/src/github_runner_image_builder/chroot.py b/src/github_runner_image_builder/chroot.py index 5ef96d1..6946733 100644 --- a/src/github_runner_image_builder/chroot.py +++ b/src/github_runner_image_builder/chroot.py @@ -35,6 +35,7 @@ def __init__(self, chroot_path: Path): Args: chroot_path: The path to set as new root. + mount_dir: Path to chroot mount dir. """ self.chroot_path = chroot_path self.root: None | int = None diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py index 27d0922..ccffdfc 100644 --- a/src/github_runner_image_builder/errors.py +++ b/src/github_runner_image_builder/errors.py @@ -29,10 +29,6 @@ class BuildImageError(ImageBuilderBaseError): """Represents an error while building the image.""" -class CleanBuildStateError(BuildImageError): - """Represents an error cleaning up build state.""" - - class BaseImageDownloadError(BuildImageError): """Represents an error downloading base image.""" @@ -41,8 +37,8 @@ class ImageResizeError(BuildImageError): """Represents an error while resizing the image.""" -class ImageMountError(BuildImageError): - """Represents an error while mounting the image to network block device.""" +class ImageConnectError(BuildImageError): + """Represents an error while connecting the image to network block device.""" class ResizePartitionError(BuildImageError): diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 8157a59..d6efcdb 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -23,7 +23,7 @@ CleanBuildStateError, DependencyInstallError, ImageCompressError, - ImageMountError, + ImageConnectError, ImageResizeError, NetworkBlockDeviceError, PermissionConfigurationError, @@ -79,7 +79,7 @@ def test_subprocess_call_funcs( pytest.param( "_mount_image_to_network_block_device", [MagicMock()], - builder.ImageMountError, + builder.ImageConnectError, id="mount image to nbd", ), pytest.param( @@ -580,8 +580,8 @@ def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPat MagicMock(side_effect=subprocess.CalledProcessError(1, [], "", "error mounting")), ) - with pytest.raises(ImageMountError) as exc: - builder._mount_image_to_network_block_device(image_path=MagicMock()) + with pytest.raises(ImageConnectError) as exc: + builder._connect_image_to_network_block_device(image_path=MagicMock()) assert "error mounting" in str(exc.getrepr()) @@ -598,7 +598,7 @@ def test__mount_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): builder, "_mount_network_block_device_partition", (mount_mock := MagicMock()) ) - builder._mount_image_to_network_block_device(image_path=MagicMock()) + builder._connect_image_to_network_block_device(image_path=MagicMock()) run_mock.assert_called_once() mount_mock.assert_called_once() From 3d4951d00ec78dabd90f32c150abc0f7e41250c7 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 10:13:03 +0000 Subject: [PATCH 33/63] separate disconnect --- src/github_runner_image_builder/builder.py | 10 +-- src/github_runner_image_builder/chroot.py | 1 - tests/unit/test_builder.py | 97 +++++++++++----------- 3 files changed, 50 insertions(+), 58 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 2f5786a..22dce19 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -646,25 +646,21 @@ def _install_yarn() -> None: raise YarnInstallError from exc -def _disconnect_image_to_network_block_device(): +def _disconnect_image_to_network_block_device() -> None: """Disconnect the image to network block device in cleanup for chroot. Raises: ImageConnectError: If there was an error disconnecting the image from network block device. """ try: - output = subprocess.run( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], timeout=30, - check=True, - capture_output=True, ) logger.info("qemu-nbd disconnect nbd out: %s", output) - output = subprocess.run( # nosec: B603 + output = subprocess.check_output( # nosec: B603 ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], timeout=30, - check=True, - capture_output=True, ) logger.info("qemu-nbd disconnect nbdp1 out: %s", output) except subprocess.CalledProcessError as exc: diff --git a/src/github_runner_image_builder/chroot.py b/src/github_runner_image_builder/chroot.py index 6946733..5ef96d1 100644 --- a/src/github_runner_image_builder/chroot.py +++ b/src/github_runner_image_builder/chroot.py @@ -35,7 +35,6 @@ def __init__(self, chroot_path: Path): Args: chroot_path: The path to set as new root. - mount_dir: Path to chroot mount dir. """ self.chroot_path = chroot_path self.root: None | int = None diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index d6efcdb..303cf94 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -20,7 +20,6 @@ BaseImageDownloadError, BuildImageError, ChrootBaseError, - CleanBuildStateError, DependencyInstallError, ImageCompressError, ImageConnectError, @@ -72,15 +71,12 @@ def test_subprocess_call_funcs( @pytest.mark.parametrize( "func, args, exc", [ - pytest.param( - "_clean_build_state", [], builder.CleanBuildStateError, id="clean build state" - ), pytest.param("_resize_image", [MagicMock()], builder.ImageResizeError, id="resize image"), pytest.param( - "_mount_image_to_network_block_device", + "_connect_image_to_network_block_device", [MagicMock()], builder.ImageConnectError, - id="mount image to nbd", + id="connect image to nbd", ), pytest.param( "_resize_mount_partitions", [], builder.ResizePartitionError, id="resize mount parts" @@ -110,6 +106,12 @@ def test_subprocess_call_funcs( builder.YarnInstallError, id="install yarn", ), + pytest.param( + "_disconnect_image_to_network_block_device", + [], + builder.ImageConnectError, + id="disconnect image to nbd", + ), pytest.param( "_compress_image", [MagicMock()], @@ -235,10 +237,9 @@ def test_build_image_error( assert: BuildImageError is raised. """ monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", MagicMock()) - monkeypatch.setattr(builder, "_clean_build_state", MagicMock()) monkeypatch.setattr(builder, "_download_and_validate_image", MagicMock()) monkeypatch.setattr(builder, "_resize_image", MagicMock()) - monkeypatch.setattr(builder, "_mount_image_to_network_block_device", MagicMock()) + monkeypatch.setattr(builder, "_connect_image_to_network_block_device", MagicMock()) monkeypatch.setattr(builder, "_resize_mount_partitions", MagicMock()) monkeypatch.setattr(builder, "_replace_mounted_resolv_conf", MagicMock()) monkeypatch.setattr(builder, "_install_yq", MagicMock()) @@ -247,6 +248,7 @@ def test_build_image_error( monkeypatch.setattr(builder, "_disable_unattended_upgrades", MagicMock()) monkeypatch.setattr(builder, "_configure_system_users", MagicMock()) monkeypatch.setattr(builder, "_install_yarn", MagicMock()) + monkeypatch.setattr(builder, "_disconnect_image_to_network_block_device", MagicMock()) monkeypatch.setattr(builder, "_compress_image", MagicMock()) monkeypatch.setattr(patch_obj, sub_func, mock) @@ -256,39 +258,6 @@ def test_build_image_error( assert expected_message in str(exc.getrepr()) -def test__clean_build_state_error(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a magic mocked IMAGE_MOUNT_DIR and subprocess call that raises exceptions. - act: when _clean_build_state is called. - assert: CleanBuildStateError is raised. - """ - mock_mount_dir = MagicMock() - mock_subprocess_run = MagicMock( - side_effect=subprocess.CalledProcessError(1, [], "", "qemu-nbd error") - ) - monkeypatch.setattr(builder, "IMAGE_MOUNT_DIR", mock_mount_dir) - monkeypatch.setattr(subprocess, "run", mock_subprocess_run) - - with pytest.raises(CleanBuildStateError) as exc: - builder._clean_build_state() - - assert "qemu-nbd error" in str(exc.getrepr()) - - -def test__clean_build_state(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given a magic mocked IMAGE_MOUNT_DIR and qemu-nbd subprocess call. - act: when _clean_build_state is called. - assert: the mocks are called. - """ - mock_subprocess_run = MagicMock() - monkeypatch.setattr(builder.subprocess, "run", mock_subprocess_run) - - builder._clean_build_state() - - mock_subprocess_run.assert_called() - - @pytest.mark.parametrize( "patch_obj, sub_func, exception, expected_message", [ @@ -568,10 +537,10 @@ def test__mount_network_block_device_partition(monkeypatch: pytest.MonkeyPatch): mock_run_call.assert_called_once() -def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): +def test__connect_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched process calls that fails. - act: when _mount_image_to_network_block_device is called. + act: when _connect_image_to_network_block_device is called. assert: ImageMountError is raised. """ monkeypatch.setattr( @@ -586,22 +555,18 @@ def test__mount_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPat assert "error mounting" in str(exc.getrepr()) -def test__mount_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): +def test__connect_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): """ arrange: given a monkeypatched mock process run calls and \ _mount_network_block_device_partition call. - act: when _mount_image_to_network_block_device is called. + act: when _connect_image_to_network_block_device is called. assert: expected calls are made. """ monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) - monkeypatch.setattr( - builder, "_mount_network_block_device_partition", (mount_mock := MagicMock()) - ) builder._connect_image_to_network_block_device(image_path=MagicMock()) - run_mock.assert_called_once() - mount_mock.assert_called_once() + run_mock.assert_called() def test__replace_mounted_resolv_conf(monkeypatch: pytest.MonkeyPatch): @@ -792,6 +757,38 @@ def test__install_yarn(monkeypatch: pytest.MonkeyPatch): assert builder._install_yarn() is None +def test__disconnect_image_to_network_block_device_fail(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched process calls that fails. + act: when _disconnect_image_to_network_block_device is called. + assert: ImageMountError is raised. + """ + monkeypatch.setattr( + subprocess, + "check_output", + MagicMock(side_effect=subprocess.CalledProcessError(1, [], "", "error mounting")), + ) + + with pytest.raises(ImageConnectError) as exc: + builder._disconnect_image_to_network_block_device() + + assert "error mounting" in str(exc.getrepr()) + + +def test__disconnect_image_to_network_block_device(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given a monkeypatched mock process run calls and \ + _mount_network_block_device_partition call. + act: when _disconnect_image_to_network_block_device is called. + assert: expected calls are made. + """ + monkeypatch.setattr(subprocess, "check_output", (check_mock := MagicMock())) + + builder._disconnect_image_to_network_block_device() + + check_mock.assert_called() + + def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): """ arrange: given subprocess run that raises CalledProcessError. From 915bf4d2b6c0099faf2b126e5db101010b2540a8 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 4 Jun 2024 14:53:11 +0000 Subject: [PATCH 34/63] failure recovery --- src/github_runner_image_builder/builder.py | 134 ++++++++++++++++----- src/github_runner_image_builder/errors.py | 4 + tests/unit/test_builder.py | 8 +- 3 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 22dce19..314ef50 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -30,6 +30,7 @@ ResizePartitionError, SystemUserConfigurationError, UnattendedUpgradeDisableError, + UnmountBuildPathError, UnsupportedArchitectureError, YarnInstallError, YQBuildError, @@ -167,6 +168,11 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: Raises: BuildImageError: If there was an error building the image. """ + # ensure clean state - if there were errors within the chroot environment (e.g. network error) + # this guarantees retry-ability + _unmount_build_path() + _disconnect_image_to_network_block_device(check=False) + IMAGE_MOUNT_DIR.mkdir(parents=True, exist_ok=True) logger.info("Downloading base image.") base_image_path = _download_and_validate_image(arch=arch, base_image=base_image) @@ -212,12 +218,109 @@ def build_image(arch: Arch, base_image: BaseImage) -> None: raise BuildImageError from exc logger.info("Disconnecting image to network block device.") - _disconnect_image_to_network_block_device() + _disconnect_image_to_network_block_device(check=True) logger.info("Compressing image.") _compress_image(base_image_path) +def _disconnect_image_to_network_block_device(check: bool = True) -> None: + """Disconnect the image to network block device in cleanup for chroot. + + Args: + check: Whether to raise an error on command failure. + + Raises: + ImageConnectError: If there was an error disconnecting the image from network block device. + """ + try: + result = subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], + check=check, + encoding="utf-8", + timeout=30, + ) + logger.info( + "qemu-nbd disconnect nbd code: %s out: %s err: %s", + result.returncode, + result.stdout, + result.stderr, + ) + result = subprocess.run( # nosec: B603 + ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + check=check, + encoding="utf-8", + timeout=30, + ) + logger.info( + "qemu-nbd disconnect nbdp1 code: %s out: %s err: %s", + result.returncode, + result.stdout, + result.stderr, + ) + except subprocess.CalledProcessError as exc: + logger.exception( + "Error disconnecting image to network block device, cmd: %s, code: %s, err: %s", + exc.cmd, + exc.returncode, + exc.output, + ) + raise ImageConnectError from exc + except subprocess.SubprocessError as exc: + raise ImageConnectError from exc + + +def _unmount_build_path() -> None: + """Unmount any mounted paths left by previous build. + + Raises: + UnmountBuildPathError: if there was an error unmounting previous build state. + """ + # The commands will fail if artefacts do not exist and hence there is no need to check the + # output of subprocess runs. + try: + output = subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "dev")], + timeout=30, + check=False, + ) # nosec: B603 + logger.info("umount dev out: %s", output) + output = subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "proc")], + timeout=30, + check=False, + capture_output=True, + ) # nosec: B603 + logger.info("umount proc out: %s", output) + output = subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR / "sys")], + timeout=30, + check=False, + capture_output=True, + ) # nosec: B603 + logger.info("umount sys out: %s", output) + output = subprocess.run( + ["/usr/bin/umount", str(IMAGE_MOUNT_DIR)], timeout=30, check=False, capture_output=True + ) # nosec: B603 + logger.info("umount ubuntu-image out: %s", output) + output = subprocess.run( + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PATH)], + timeout=30, + check=False, + capture_output=True, + ) # nosec: B603 + logger.info("umount nbd out: %s", output) + output = subprocess.run( # nosec: B603 + ["/usr/bin/umount", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], + timeout=30, + check=False, + capture_output=True, + ) + logger.info("umount nbdp1 out: %s", output) + except subprocess.SubprocessError as exc: + raise UnmountBuildPathError from exc + + def _download_and_validate_image(arch: Arch, base_image: BaseImage) -> Path: """Download and verify the base image from cloud-images.ubuntu.com. @@ -646,35 +749,6 @@ def _install_yarn() -> None: raise YarnInstallError from exc -def _disconnect_image_to_network_block_device() -> None: - """Disconnect the image to network block device in cleanup for chroot. - - Raises: - ImageConnectError: If there was an error disconnecting the image from network block device. - """ - try: - output = subprocess.check_output( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PATH)], - timeout=30, - ) - logger.info("qemu-nbd disconnect nbd out: %s", output) - output = subprocess.check_output( # nosec: B603 - ["/usr/bin/qemu-nbd", "--disconnect", str(NETWORK_BLOCK_DEVICE_PARTITION_PATH)], - timeout=30, - ) - logger.info("qemu-nbd disconnect nbdp1 out: %s", output) - except subprocess.CalledProcessError as exc: - logger.exception( - "Error disconnecting image to network block device, cmd: %s, code: %s, err: %s", - exc.cmd, - exc.returncode, - exc.output, - ) - raise ImageConnectError from exc - except subprocess.SubprocessError as exc: - raise ImageConnectError from exc - - # Image compression might fail for arbitrary reasons - retrying usually solves this. @retry(tries=5, delay=5, max_delay=60, backoff=2, local_logger=logger) def _compress_image(image: Path) -> None: diff --git a/src/github_runner_image_builder/errors.py b/src/github_runner_image_builder/errors.py index ccffdfc..81e0666 100644 --- a/src/github_runner_image_builder/errors.py +++ b/src/github_runner_image_builder/errors.py @@ -29,6 +29,10 @@ class BuildImageError(ImageBuilderBaseError): """Represents an error while building the image.""" +class UnmountBuildPathError(BuildImageError): + """Represents an error while unmounting build path.""" + + class BaseImageDownloadError(BuildImageError): """Represents an error downloading base image.""" diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 303cf94..5e93ad3 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -41,6 +41,7 @@ @pytest.mark.parametrize( "func, args", [ + pytest.param("_unmount_build_path", [], id="unmount build path"), pytest.param("_install_dependencies", [], id="install dependencies"), pytest.param("_enable_network_block_device", [], id="enable network block device"), pytest.param("_resize_image", [MagicMock()], id="resize image"), @@ -71,6 +72,9 @@ def test_subprocess_call_funcs( @pytest.mark.parametrize( "func, args, exc", [ + pytest.param( + "_unmount_build_path", [], builder.UnmountBuildPathError, id="unmount build path" + ), pytest.param("_resize_image", [MagicMock()], builder.ImageResizeError, id="resize image"), pytest.param( "_connect_image_to_network_block_device", @@ -765,7 +769,7 @@ def test__disconnect_image_to_network_block_device_fail(monkeypatch: pytest.Monk """ monkeypatch.setattr( subprocess, - "check_output", + "run", MagicMock(side_effect=subprocess.CalledProcessError(1, [], "", "error mounting")), ) @@ -782,7 +786,7 @@ def test__disconnect_image_to_network_block_device(monkeypatch: pytest.MonkeyPat act: when _disconnect_image_to_network_block_device is called. assert: expected calls are made. """ - monkeypatch.setattr(subprocess, "check_output", (check_mock := MagicMock())) + monkeypatch.setattr(subprocess, "run", (check_mock := MagicMock())) builder._disconnect_image_to_network_block_device() From 06dcc0c8b5b25782529d1f90736f1bf24dd85e18 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 6 Jun 2024 05:30:51 +0000 Subject: [PATCH 35/63] chore: add retry to network related funcs --- src/github_runner_image_builder/builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 314ef50..01a0456 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -378,6 +378,7 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") +@retry(tries=3, delay=5, max_delay=30, backoff=2, local_logger=logger) def _download_base_image(base_image: BaseImage, bin_arch: str, output_filename: str) -> Path: """Download the base image. @@ -406,6 +407,7 @@ def _download_base_image(base_image: BaseImage, bin_arch: str, output_filename: return Path(output_filename) +@retry(tries=3, delay=5, max_delay=30, backoff=2, local_logger=logger) def _fetch_shasums(base_image: BaseImage) -> dict[str, str]: """Fetch SHA256SUM for given base image. @@ -557,6 +559,7 @@ def _resize_mount_partitions() -> None: raise ResizePartitionError from exc +@retry(tries=3, delay=5, max_delay=30, backoff=2, local_logger=logger) def _install_yq() -> None: """Build and install yq from source. From 7094592bdf475ba7e442623ff3c929c06ea1ae69 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 6 Jun 2024 05:43:43 +0000 Subject: [PATCH 36/63] fix: unit tests --- src/github_runner_image_builder/utils.py | 1 - tests/unit/test_builder.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/github_runner_image_builder/utils.py b/src/github_runner_image_builder/utils.py index 47d8980..1568944 100644 --- a/src/github_runner_image_builder/utils.py +++ b/src/github_runner_image_builder/utils.py @@ -72,7 +72,6 @@ def fn_with_retry(*args: ParamT.args, **kwargs: ParamT.kwargs) -> ReturnT: for _ in range(tries): try: - print("Trying") return func(*args, **kwargs) # Error caught is set by the input of the function. except exception as err: # pylint: disable=broad-exception-caught diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 5e93ad3..74ac20f 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -413,6 +413,8 @@ def test__download_base_image_error(monkeypatch: pytest.MonkeyPatch): act: when _download_base_image is called. assert: BaseImageDownloadError is raised. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr( builder.urllib.request, "urlretrieve", @@ -447,6 +449,8 @@ def test__fetch_shasums_error(monkeypatch: pytest.MonkeyPatch): act: when _fetch_shasums is called. assert: BaseImageDownloadError is raised. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr( builder.requests, "get", @@ -465,6 +469,8 @@ def test__fetch_shasums(monkeypatch: pytest.MonkeyPatch): act: when _fetch_shasums is called. assert: a dictionary with filename to shasum is created. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) mock_response = MagicMock() mock_response.content = bytes( """test_shasum1 *file1 @@ -614,10 +620,16 @@ def test__install_yq_error(monkeypatch: pytest.MonkeyPatch): act: when _install_yq is called. assert: YQBuildError is raised. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr( subprocess, "check_output", - MagicMock(side_effect=[None, subprocess.CalledProcessError(1, [], "", "Go build error.")]), + MagicMock( + # tried 3 times via retry + side_effect=[None, subprocess.CalledProcessError(1, [], "", "Go build error.")] + * 3 + ), ) with pytest.raises(YQBuildError) as exc: @@ -632,6 +644,8 @@ def test__install_yq_already_exists(monkeypatch: pytest.MonkeyPatch): act: when _install_yq is called. assert: Mock functions are called. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr(builder, "YQ_REPOSITORY_PATH", MagicMock(return_value=True)) monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) @@ -648,6 +662,8 @@ def test__install_yq(monkeypatch: pytest.MonkeyPatch): act: when _install_yq is called. assert: Mock functions are called. """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) monkeypatch.setattr(subprocess, "check_output", (run_mock := MagicMock())) monkeypatch.setattr(shutil, "copy", (copy_mock := MagicMock())) From ea325ebce45dc2f554050f5e8fc3a43187a5b1fc Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sat, 8 Jun 2024 11:27:28 +0000 Subject: [PATCH 37/63] feat: arm build w/ no kvm --- src/github_runner_image_builder/builder.py | 16 ++++++- tests/unit/test_builder.py | 50 +++++++++++++++++++--- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index 01a0456..d7ae778 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -46,6 +46,7 @@ "libguestfs-tools", # used to modify VM images. "cloud-utils", # used for growpart. "golang-go", # used to build yq from source. + "cpu-checker", # used to check KVM capabilities for image compression ] APT_NONINTERACTIVE_ENV = {"DEBIAN_FRONTEND": "noninteractive"} SNAP_GO = "go" @@ -763,9 +764,22 @@ def _compress_image(image: Path) -> None: Raises: ImageCompressError: If there was something wrong compressing the image. """ + try: + subprocess.run(["/usr/sbin/kvm-ok"], check=True, encoding="utf-8") # nosec: B603 + except subprocess.SubprocessError: + logger.info("KVM capability not found, skipping compression.") + image.rename(str(IMAGE_OUTPUT_PATH)) + return + try: output = subprocess.check_output( # nosec: B603 - ["/usr/bin/virt-sparsify", "--compress", str(image), str(IMAGE_OUTPUT_PATH)], + [ + "/usr/bin/sudo", + "/usr/bin/virt-sparsify", + "--compress", + str(image), + str(IMAGE_OUTPUT_PATH), + ], timeout=60 * 10, ) logger.info("virt-sparsify compress out: %s", output) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 74ac20f..cb7a572 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -116,12 +116,6 @@ def test_subprocess_call_funcs( builder.ImageConnectError, id="disconnect image to nbd", ), - pytest.param( - "_compress_image", - [MagicMock()], - builder.ImageCompressError, - id="compress image", - ), ], ) def test_subprocess_func_errors( @@ -817,6 +811,9 @@ def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): """ # Bypass decorated retry sleep monkeypatch.setattr(time, "sleep", MagicMock()) + monkeypatch.setattr( + subprocess, "run", MagicMock(return_value=subprocess.CompletedProcess([], 0, "", "")) + ) monkeypatch.setattr( subprocess, "check_output", @@ -827,3 +824,44 @@ def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): builder._compress_image(image=MagicMock()) assert "Compression error" in str(exc.getrepr()) + + +def test__compress_image_no_kvm(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given subprocess run for kvm-ok that raises an error. + act: when _compress_image is called. + assert: image is renamed. + """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) + monkeypatch.setattr( + subprocess, + "run", + MagicMock(side_effect=subprocess.CalledProcessError(1, [], "kvm module not enabled")), + ) + image_mock = MagicMock() + + builder._compress_image(image=image_mock) + image_mock.rename.assert_called_once() + + +def test__compress_image_subprocess_error(monkeypatch: pytest.MonkeyPatch): + """ + arrange: given subprocess check_output raises an error. + act: when _compress_image is called. + assert: ImageCompressError is raised. + """ + # Bypass decorated retry sleep + monkeypatch.setattr(time, "sleep", MagicMock()) + monkeypatch.setattr(subprocess, "run", MagicMock()) + monkeypatch.setattr( + subprocess, + "check_output", + MagicMock(side_effect=subprocess.SubprocessError("Image subprocess err")), + ) + image_mock = MagicMock() + + with pytest.raises(builder.ImageCompressError) as exc: + builder._compress_image(image=image_mock) + + assert "Image subprocess err" in str(exc.getrepr()) From 7c20ca54f8684c23d49b9fd7e1a04aad50aeed0b Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Sat, 8 Jun 2024 16:36:27 +0000 Subject: [PATCH 38/63] feat: platform independent compression --- src/github_runner_image_builder/builder.py | 22 ++++++++++------------ tests/unit/test_builder.py | 19 ------------------- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/src/github_runner_image_builder/builder.py b/src/github_runner_image_builder/builder.py index d7ae778..1e31e90 100644 --- a/src/github_runner_image_builder/builder.py +++ b/src/github_runner_image_builder/builder.py @@ -2,6 +2,8 @@ # See LICENSE file for licensing details. """Module for interacting with qemu image builder.""" +# nosec: B603 is added throughout subprocess calls, make sure that they are running trusted user +# inputs. import hashlib import logging @@ -43,10 +45,8 @@ APT_DEPENDENCIES = [ "qemu-utils", # used for qemu utilities tools to build and resize image - "libguestfs-tools", # used to modify VM images. "cloud-utils", # used for growpart. "golang-go", # used to build yq from source. - "cpu-checker", # used to check KVM capabilities for image compression ] APT_NONINTERACTIVE_ENV = {"DEBIAN_FRONTEND": "noninteractive"} SNAP_GO = "go" @@ -764,25 +764,23 @@ def _compress_image(image: Path) -> None: Raises: ImageCompressError: If there was something wrong compressing the image. """ - try: - subprocess.run(["/usr/sbin/kvm-ok"], check=True, encoding="utf-8") # nosec: B603 - except subprocess.SubprocessError: - logger.info("KVM capability not found, skipping compression.") - image.rename(str(IMAGE_OUTPUT_PATH)) - return - try: output = subprocess.check_output( # nosec: B603 [ "/usr/bin/sudo", - "/usr/bin/virt-sparsify", - "--compress", + "/usr/bin/qemu-img", + "convert", + "-c", # compress + "-f", # input format + "qcow2", + "-O", # output format + "qcow2", str(image), str(IMAGE_OUTPUT_PATH), ], timeout=60 * 10, ) - logger.info("virt-sparsify compress out: %s", output) + logger.info("qemu-img convert compress out: %s", output) except subprocess.CalledProcessError as exc: logger.exception( "Error compressing image, cmd: %s, code: %s, err: %s", diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index cb7a572..e33732c 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -826,25 +826,6 @@ def test__compress_image_fail(monkeypatch: pytest.MonkeyPatch): assert "Compression error" in str(exc.getrepr()) -def test__compress_image_no_kvm(monkeypatch: pytest.MonkeyPatch): - """ - arrange: given subprocess run for kvm-ok that raises an error. - act: when _compress_image is called. - assert: image is renamed. - """ - # Bypass decorated retry sleep - monkeypatch.setattr(time, "sleep", MagicMock()) - monkeypatch.setattr( - subprocess, - "run", - MagicMock(side_effect=subprocess.CalledProcessError(1, [], "kvm module not enabled")), - ) - image_mock = MagicMock() - - builder._compress_image(image=image_mock) - image_mock.rename.assert_called_once() - - def test__compress_image_subprocess_error(monkeypatch: pytest.MonkeyPatch): """ arrange: given subprocess check_output raises an error. From 013aaee25ed160c3318af90170e9b65452128539 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 11 Jun 2024 14:08:32 +0000 Subject: [PATCH 39/63] non-vm test --- tests/integration/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 8f357e1..0de6ef7 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -198,7 +198,6 @@ async def create_lxd_instance(lxd_client: Client, image: str) -> Instance: instance_config = { "name": f"test-{image}", "source": {"type": "image", "alias": image}, - "type": "virtual-machine", "config": {"limits.cpu": "3", "limits.memory": "8192MiB"}, } instance: Instance = lxd_client.instances.create( # pylint: disable=no-member From d93aa4e0c13e966d414f001717ae37cd51bd3371 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Tue, 11 Jun 2024 14:31:11 +0000 Subject: [PATCH 40/63] lxd use virtual machine --- tests/integration/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 0de6ef7..8f357e1 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -198,6 +198,7 @@ async def create_lxd_instance(lxd_client: Client, image: str) -> Instance: instance_config = { "name": f"test-{image}", "source": {"type": "image", "alias": image}, + "type": "virtual-machine", "config": {"limits.cpu": "3", "limits.memory": "8192MiB"}, } instance: Instance = lxd_client.instances.create( # pylint: disable=no-member From d2214dabdc19fcd58e7ce34bde4d33719bb3f34c Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 02:30:32 +0000 Subject: [PATCH 41/63] test: openstack runner test (arm64 cannot launch lxd due to kvm) --- .github/workflows/integration_test.yaml | 4 +- tests/integration/conftest.py | 154 +++++++++++++++++++++- tests/integration/helpers.py | 166 +++++++++++++++++++----- tests/integration/requirements.txt | 2 + tests/integration/test_image.py | 21 ++- tests/integration/types.py | 57 ++++++++ tox.ini | 5 + 7 files changed, 370 insertions(+), 39 deletions(-) create mode 100644 tests/integration/types.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 94e9a2a..a40d51c 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -25,7 +25,7 @@ jobs: pipx install tox # need to run in sudo mode due to chroot - name: Run integration tests - run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} + run: sudo $(which tox) -e integration -- -m arm64 --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} integration-tests-amd: name: Integration test (X64) @@ -44,4 +44,4 @@ jobs: pipx install tox # need to run in sudo mode due to chroot - name: Run integration tests - run: sudo $(which tox) -e integration -- --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} + run: sudo $(which tox) -e integration -- -m amd64 --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ca2c446..1ce1c28 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,17 +2,27 @@ # See LICENSE file for licensing details. """Fixtures for github runner image builder integration tests.""" +import logging +import secrets import string +import typing from pathlib import Path -from typing import Optional import openstack import pytest +import pytest_asyncio import yaml +from fabric.connection import Connection as SSHConnection +from openstack.compute.v2.keypair import Keypair +from openstack.compute.v2.server import Server from openstack.connection import Connection from openstack.image.v2.image import Image +from openstack.network.v2.security_group import SecurityGroup from github_runner_image_builder.cli import main +from tests.integration import helpers, types + +logger = logging.getLogger(__name__) @pytest.fixture(scope="module", name="image") @@ -31,7 +41,7 @@ def openstack_clouds_yaml_fixture(pytestconfig: pytest.Config) -> str: @pytest.fixture(scope="module", name="private_endpoint_clouds_yaml") -def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> Optional[str]: +def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> typing.Optional[str]: """The openstack private endpoint clouds yaml.""" auth_url = pytestconfig.getoption("--openstack-auth-url") password = pytestconfig.getoption("--openstack-password") @@ -70,7 +80,7 @@ def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> Optiona @pytest.fixture(scope="module", name="clouds_yaml_contents") def clouds_yaml_contents_fixture( - openstack_clouds_yaml: Optional[str], private_endpoint_clouds_yaml: Optional[str] + openstack_clouds_yaml: typing.Optional[str], private_endpoint_clouds_yaml: typing.Optional[str] ): """The Openstack clouds yaml or private endpoint cloud yaml contents.""" clouds_yaml_contents = openstack_clouds_yaml or private_endpoint_clouds_yaml @@ -120,10 +130,144 @@ def callback_script_fixture(callback_result_path: Path) -> Path: return callback_script +@pytest.fixture(scope="module", name="test_id") +def test_id_fixture() -> str: + """The unique test identifier.""" + return secrets.token_hex(4) + + @pytest.fixture(scope="module", name="openstack_image_name") -def openstack_image_name_fixture() -> str: +def openstack_image_name_fixture(test_id: str) -> str: """The image name to upload to openstack.""" - return "image-builder-test-image" + return f"image-builder-test-image-{test_id}" + + +@pytest.fixture(scope="module", name="ssh_key") +def ssh_key_fixture( + openstack_connection: Connection, test_id: str +) -> typing.Generator[types.SSHKey, None, None]: + """The openstack ssh key fixture.""" + keypair: Keypair = openstack_connection.create_keypair(f"test-image-builder-keys-{test_id}") + ssh_key_path = Path("tmp_key") + ssh_key_path.touch(exist_ok=True) + ssh_key_path.write_text(keypair.private_key, encoding="utf-8") + + yield types.SSHKey(keypair=keypair, private_key=ssh_key_path) + + openstack_connection.delete_keypair(name=keypair.name) + + +class OpenstackMeta(typing.NamedTuple): + """A wrapper around Openstack related info. + + Attributes: + connection: The connection instance to Openstack. + ssh_key: The SSH-Key created to connect to Openstack instance. + network: The Openstack network to create instances under. + flavor: The flavor to create instances with. + """ + + connection: Connection + ssh_key: types.SSHKey + network: str + flavor: str + + +@pytest.fixture(scope="module", name="openstack_metadata") +def openstack_metadata_fixture( + openstack_connection: Connection, ssh_key: types.SSHKey, network_name: str, flavor_name: str +) -> OpenstackMeta: + """A wrapper around openstack related info.""" + return OpenstackMeta( + connection=openstack_connection, ssh_key=ssh_key, network=network_name, flavor=flavor_name + ) + + +@pytest.fixture(scope="module", name="openstack_security_group") +def openstack_security_group_fixture(openstack_connection: Connection): + """An ssh-connectable security group.""" + security_group_name = "github-runner-image-builder-operator-test-security-group" + security_group: SecurityGroup = openstack_connection.create_security_group( + name=security_group_name, + description="For servers managed by the github-runner-image-builder charm.", + ) + # For ping + openstack_connection.create_security_group_rule( + secgroup_name_or_id=security_group_name, + protocol="icmp", + direction="ingress", + ethertype="IPv4", + ) + # For SSH + openstack_connection.create_security_group_rule( + secgroup_name_or_id=security_group_name, + port_range_min="22", + port_range_max="22", + protocol="tcp", + direction="ingress", + ethertype="IPv4", + ) + # For tmate + openstack_connection.create_security_group_rule( + secgroup_name_or_id=security_group_name, + port_range_min="10022", + port_range_max="10022", + protocol="tcp", + direction="egress", + ethertype="IPv4", + ) + + yield security_group + + openstack_connection.delete_security_group(security_group_name) + + +@pytest_asyncio.fixture(scope="module", name="openstack_server") +async def openstack_server_fixture( + openstack_metadata: OpenstackMeta, + openstack_security_group: SecurityGroup, + openstack_image_name: str, + test_id: str, +): + """A testing openstack instance.""" + server_name = f"test-server-{test_id}" + images: list[Image] = openstack_metadata.connection.search_images(openstack_image_name) + assert images, "No built image found." + server: Server = openstack_metadata.connection.create_server( + name=server_name, + image=images[0], + key_name=openstack_metadata.ssh_key.keypair.name, + auto_ip=False, + # these are pre-configured values on private endpoint. + security_groups=[openstack_security_group.name], + flavor=openstack_metadata.flavor, + network=openstack_metadata.network, + timeout=120, + wait=True, + ) + + yield server + + openstack_metadata.connection.delete_server(server_name, wait=True) + for image in images: + openstack_metadata.connection.delete_image(image.id) + + +@pytest_asyncio.fixture(scope="module", name="ssh_connection") +async def ssh_connection_fixture( + openstack_server: Server, openstack_metadata: OpenstackMeta, proxy: types.ProxyConfig +) -> SSHConnection: + """The openstack server ssh connection fixture.""" + logger.info("Setting up SSH connection.") + ssh_connection = helpers.wait_for_valid_connection( + connection=openstack_metadata.connection, + server_name=openstack_server.name, + network=openstack_metadata.network, + ssh_key=openstack_metadata.ssh_key.private_key, + proxy=proxy, + ) + + return ssh_connection @pytest.fixture(scope="module", name="cli_run") diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 8f357e1..c96033f 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -5,6 +5,7 @@ import collections import inspect +import logging import platform import tarfile import time @@ -13,11 +14,20 @@ from string import Template from typing import Awaitable, Callable, ParamSpec, TypeVar, cast +from fabric import Connection as SSHConnection +from fabric import Result +from openstack.compute.v2.server import Server +from openstack.connection import Connection +from paramiko.ssh_exception import NoValidConnectionsError from pylxd import Client from pylxd.models.image import Image from pylxd.models.instance import Instance, InstanceState from requests_toolbelt import MultipartEncoder +from tests.integration import types + +logger = logging.getLogger(__name__) + P = ParamSpec("P") R = TypeVar("R") S = Callable[P, R] | Callable[P, Awaitable[R]] @@ -62,6 +72,31 @@ async def wait_for( raise TimeoutError() +def create_lxd_vm_image(lxd_client: Client, img_path: Path, image: str, tmp_path: Path) -> Image: + """Create LXD VM image. + + 1. Creates the metadata.tar.gz file with the corresponding Ubuntu OS image and a pre-defined + templates directory. See testdata/templates. + 2. Uploads the created VM image to LXD - metadata and image of qcow2 format is required. + 3. Tags the uploaded image with an alias for test use. + + Args: + lxd_client: PyLXD client. + img_path: qcow2 (.img) file path to upload. + tmp_path: Temporary dir. + image: The Ubuntu image name. + + Returns: + The created LXD image. + """ + metadata_tar = _create_metadata_tar_gz(image=image, tmp_path=tmp_path) + lxd_image = _post_vm_img( + lxd_client, img_path.read_bytes(), metadata_tar.read_bytes(), public=True + ) + lxd_image.add_alias(image, f"Ubuntu {image} {IMAGE_TO_TAG[image]} image.") + return lxd_image + + IMAGE_TO_TAG = {"jammy": "22.04", "noble": "24.04"} @@ -139,29 +174,29 @@ def _post_vm_img( return Image(client, fingerprint=operation.metadata["fingerprint"]) -def create_lxd_vm_image(lxd_client: Client, img_path: Path, image: str, tmp_path: Path) -> Image: - """Create LXD VM image. - - 1. Creates the metadata.tar.gz file with the corresponding Ubuntu OS image and a pre-defined - templates directory. See testdata/templates. - 2. Uploads the created VM image to LXD - metadata and image of qcow2 format is required. - 3. Tags the uploaded image with an alias for test use. +async def create_lxd_instance(lxd_client: Client, image: str) -> Instance: + """Create and wait for LXD instance to become active. Args: lxd_client: PyLXD client. - img_path: qcow2 (.img) file path to upload. - tmp_path: Temporary dir. image: The Ubuntu image name. Returns: - The created LXD image. + The created and running LXD instance. """ - metadata_tar = _create_metadata_tar_gz(image=image, tmp_path=tmp_path) - lxd_image = _post_vm_img( - lxd_client, img_path.read_bytes(), metadata_tar.read_bytes(), public=True + instance_config = { + "name": f"test-{image}", + "source": {"type": "image", "alias": image}, + "type": "virtual-machine", + "config": {"limits.cpu": "3", "limits.memory": "8192MiB"}, + } + instance: Instance = lxd_client.instances.create( # pylint: disable=no-member + instance_config, wait=True ) - lxd_image.add_alias(image, f"Ubuntu {image} {IMAGE_TO_TAG[image]} image.") - return lxd_image + instance.start(timeout=10 * 60, wait=True) + await wait_for(partial(_instance_running, instance)) + + return instance def _instance_running(instance: Instance) -> bool: @@ -185,26 +220,95 @@ def _instance_running(instance: Instance) -> bool: return result.exit_code == 0 -async def create_lxd_instance(lxd_client: Client, image: str) -> Instance: - """Create and wait for LXD instance to become active. +# All the arguments are necessary +def wait_for_valid_connection( # pylint: disable=too-many-arguments + connection: Connection, + server_name: str, + network: str, + ssh_key: Path, + timeout: int = 30 * 60, + proxy: types.ProxyConfig | None = None, +) -> SSHConnection: + """Wait for a valid SSH connection from Openstack server. Args: - lxd_client: PyLXD client. - image: The Ubuntu image name. + connection: The openstack connection client to communicate with Openstack. + server_name: Openstack server to find the valid connection from. + network: The network to find valid connection from. + ssh_key: The path to public ssh_key to create connection with. + timeout: Number of seconds to wait before raising a timeout error. + proxy: The proxy to configure on host runner. + + Raises: + TimeoutError: If no valid connections were found. Returns: - The created and running LXD instance. + SSHConnection. """ - instance_config = { - "name": f"test-{image}", - "source": {"type": "image", "alias": image}, - "type": "virtual-machine", - "config": {"limits.cpu": "3", "limits.memory": "8192MiB"}, + start_time = time.time() + while time.time() - start_time <= timeout: + server: Server | None = connection.get_server(name_or_id=server_name) + if not server or not server.addresses: + time.sleep(10) + continue + for address in server.addresses[network]: + ip = address["addr"] + logger.info("Trying SSH into %s using key: %s...", ip, str(ssh_key.absolute())) + ssh_connection = SSHConnection( + host=ip, + user="ubuntu", + connect_kwargs={"key_filename": str(ssh_key.absolute())}, + connect_timeout=10, + ) + try: + result: Result = ssh_connection.run("echo 'hello world'") + if result.ok: + _install_proxy(conn=ssh_connection, proxy=proxy) + return ssh_connection + except NoValidConnectionsError as exc: + logger.warning("Connection not yet ready, %s.", str(exc)) + time.sleep(10) + raise TimeoutError("No valid ssh connections found.") + + +def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): + """Run commands to install proxy. + + Args: + conn: The SSH connection instance. + proxy: The proxy to apply if available. + """ + if not proxy or not proxy.http: + return + command = "sudo snap install aproxy --edge" + logger.info("Running command: %s", command) + result: Result = conn.run(command) + assert result.ok, "Failed to install aproxy" + + proxy_str = proxy.http.replace("http://", "").replace("https://", "") + command = f"sudo snap set aproxy proxy={proxy_str}" + logger.info("Running command: %s", command) + result = conn.run(command) + assert result.ok, "Failed to setup aproxy" + + # ignore line too long since it is better read without line breaks + command = """/usr/bin/sudo nft -f - << EOF +define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \\K\\S+') | grep -oP 'src \\K\\S+') +define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } +table ip aproxy +flush table ip aproxy +table ip aproxy { + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + ip daddr != \\$private-ips tcp dport { 80, 443 } counter dnat to \\$default-ip:8443 } - instance: Instance = lxd_client.instances.create( # pylint: disable=no-member - instance_config, wait=True - ) - instance.start(timeout=10 * 60, wait=True) - await wait_for(partial(_instance_running, instance)) - return instance + chain output { + type nat hook output priority -100; policy accept; + ip daddr != \\$private-ips tcp dport { 80, 443 } counter dnat to \\$default-ip:8443 + } +} +EOF""" # noqa: E501 + logger.info("Running command: %s", command) + result = conn.run(command) + assert result.ok, "Failed to configure iptable rules" diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 0c7f3b3..16911e9 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1 +1,3 @@ pylxd +fabric +types-paramiko diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index ee10ef6..71de102 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -8,6 +8,8 @@ from typing import NamedTuple import pytest +from fabric.connection import Connection as SSHConnection +from fabric.runners import Result from openstack.connection import Connection from pylxd import Client @@ -69,8 +71,9 @@ class Commands(NamedTuple): @pytest.mark.asyncio +@pytest.mark.amd64 @pytest.mark.usefixtures("cli_run") -async def test_image(image: str, tmp_path: Path): +async def test_image_amd(image: str, tmp_path: Path): """ arrange: given a built output from the CLI. act: when the image is booted and commands are executed. @@ -105,6 +108,22 @@ async def test_openstack_upload(openstack_connection: Connection, openstack_imag assert len(openstack_connection.search_images(openstack_image_name)) +@pytest.mark.asyncio +@pytest.mark.arm64 +@pytest.mark.usefixtures("cli_run") +async def test_image_arm(ssh_connection: SSHConnection): + """ + arrange: given a built output from the CLI. + act: when the image is booted and commands are executed. + assert: commands do not error. + """ + for testcmd in TEST_RUNNER_COMMANDS: + logger.info("Running command: %s", testcmd.command) + result: Result = ssh_connection.run(testcmd.command) + logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) + assert result.return_code == 0 + + @pytest.mark.asyncio @pytest.mark.usefixtures("cli_run") async def test_script_callback(callback_result_path: Path): diff --git a/tests/integration/types.py b/tests/integration/types.py new file mode 100644 index 0000000..130de01 --- /dev/null +++ b/tests/integration/types.py @@ -0,0 +1,57 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Types used in the integration test.""" + +import typing +from pathlib import Path + +from openstack.compute.v2.keypair import Keypair + + +class ProxyConfig(typing.NamedTuple): + """Proxy configuration. + + Attributes: + http: HTTP proxy address. + https: HTTPS proxy address. + no_proxy: Comma-separated list of hosts that should not be proxied. + """ + + http: str + https: str + no_proxy: str + + +class SSHKey(typing.NamedTuple): + """Openstack SSH Keypair and private key. + + Attributes: + keypair: OpenStach SSH Keypair object. + private_key: The path to private key. + """ + + keypair: Keypair + private_key: Path + + +class PrivateEndpointConfigs(typing.TypedDict): + """The Private endpoint configuration values. + + Attributes: + auth_url: OpenStack uthentication URL (Keystone). + password: OpenStack password. + project_domain_name: OpenStack project domain to use. + project_name: OpenStack project to use within the domain. + user_domain_name: OpenStack user domain to use. + username: OpenStack user to use within the domain. + region_name: OpenStack deployment region. + """ + + auth_url: str + password: str + project_domain_name: str + project_name: str + user_domain_name: str + username: str + region_name: str diff --git a/tox.ini b/tox.ini index ab4c184..6aefdad 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,11 @@ deps = commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} +[pytest] +markers = + amd64 + arm64 + [testenv:src-docs] allowlist_externals=sh description = Generate documentation for src From 4a1f739ce32975b4bb14cbfff83178b0d52803d3 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:16:45 +0000 Subject: [PATCH 42/63] test: move markers to pyproject.toml --- pyproject.toml | 4 ++++ tox.ini | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 828c81a..1f835b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,10 @@ show_missing = true [tool.pytest.ini_options] log_cli_level = "INFO" +markers = [ + "amd64", + "arm64", +] # Formatting tools configuration [tool.black] diff --git a/tox.ini b/tox.ini index 6aefdad..ab4c184 100644 --- a/tox.ini +++ b/tox.ini @@ -108,11 +108,6 @@ deps = commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -[pytest] -markers = - amd64 - arm64 - [testenv:src-docs] allowlist_externals=sh description = Generate documentation for src From cf7f00be1fea6a04ebff221418d8e3e5c3bf2665 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:16:54 +0000 Subject: [PATCH 43/63] test: add network name fixture --- tests/integration/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1ce1c28..c48e756 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -78,6 +78,14 @@ def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> typing. ) +@pytest.fixture(scope="module", name="network_name") +def network_name_fixture(pytestconfig: pytest.Config) -> str: + """Network to use to spawn test instances under.""" + network_name = pytestconfig.getoption("--openstack-network-name") + assert network_name, "Please specify the --openstack-network-name command line option" + return network_name + + @pytest.fixture(scope="module", name="clouds_yaml_contents") def clouds_yaml_contents_fixture( openstack_clouds_yaml: typing.Optional[str], private_endpoint_clouds_yaml: typing.Optional[str] From 7b491a0fdbb4d0f0890a945f1f8530acbafb0a98 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:19:46 +0000 Subject: [PATCH 44/63] test: add flavor name fixture --- tests/conftest.py | 10 ++++++++++ tests/integration/conftest.py | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index b06afc8..0dc7085 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,16 @@ def pytest_addoption(parser: Parser): parser: The pytest argument parser. """ parser.addoption("--image", action="store", help="The Ubuntu LTS base image to build.") + parser.addoption( + "--openstack-network-name", + action="store", + help="The Openstack network to create testing instances under.", + ) + parser.addoption( + "--openstack-flavor-name", + action="store", + help="The Openstack flavor to create testing instances with.", + ) parser.addoption( "--openstack-clouds-yaml", action="store", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c48e756..e42fd2f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -86,6 +86,14 @@ def network_name_fixture(pytestconfig: pytest.Config) -> str: return network_name +@pytest.fixture(scope="module", name="flavor_name") +def flavor_name_fixture(pytestconfig: pytest.Config) -> str: + """Flavor to create testing instances with.""" + flavor_name = pytestconfig.getoption("--openstack-flavor-name") + assert flavor_name, "Please specify the --openstack-flavor-name command line option" + return flavor_name + + @pytest.fixture(scope="module", name="clouds_yaml_contents") def clouds_yaml_contents_fixture( openstack_clouds_yaml: typing.Optional[str], private_endpoint_clouds_yaml: typing.Optional[str] From 605c4475c22b3c24fe878938ae2148de3e7fb198 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:30:21 +0000 Subject: [PATCH 45/63] test: fix typo --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0dc7085..5ff7b78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,7 +56,7 @@ def pytest_addoption(parser: Parser): help="The Openstack user domain name to use.", ) parser.addoption( - "--openstack-user-name", + "--openstack-username", action="store", help="The Openstack user to authenticate as.", ) From ecbf9aad27dbc9f466c42e663a6ec4cb3200ef79 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:52:56 +0000 Subject: [PATCH 46/63] test: fix typo --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e42fd2f..88193a0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -48,7 +48,7 @@ def private_endpoint_clouds_yaml_fixture(pytestconfig: pytest.Config) -> typing. project_domain_name = pytestconfig.getoption("--openstack-project-domain-name") project_name = pytestconfig.getoption("--openstack-project-name") user_domain_name = pytestconfig.getoption("--openstack-user-domain-name") - user_name = pytestconfig.getoption("--openstack-user-name") + user_name = pytestconfig.getoption("--openstack-username") region_name = pytestconfig.getoption("--openstack-region-name") if any( not val From e9764d9d1e0d77f16fc2461c60a60edb23c255d4 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 03:53:10 +0000 Subject: [PATCH 47/63] test: fix typo --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 88193a0..274976c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -103,7 +103,7 @@ def clouds_yaml_contents_fixture( assert clouds_yaml_contents, ( "Please specify --openstack-clouds-yaml or all of private endpoint arguments " "(--openstack-auth-url, --openstack-password, --openstack-project-domain-name, " - "--openstack-project-name, --openstack-user-domain-name, --openstack-user-name, " + "--openstack-project-name, --openstack-user-domain-name, --openstack-username, " "--openstack-region-name)" ) return clouds_yaml_contents From ea8080703c58f093c289dc9291fc15cbb44ed64f Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 05:31:59 +0000 Subject: [PATCH 48/63] chore: add proxy configs --- tests/conftest.py | 12 ++++++++++++ tests/integration/conftest.py | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5ff7b78..9368997 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,3 +65,15 @@ def pytest_addoption(parser: Parser): action="store", help="The Openstack region to authenticate to.", ) + parser.addoption( + "--proxy", + action="store", + help="The HTTP proxy URL to apply on the Openstack runners.", + default=None, + ) + parser.addoption( + "--no-proxy", + action="store", + help="The no proxy URL(s) to apply on the Openstack runners.", + default=None, + ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 274976c..5a217b3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -269,6 +269,14 @@ async def openstack_server_fixture( openstack_metadata.connection.delete_image(image.id) +@pytest.fixture(scope="module", name="proxy") +def proxy_fixture(pytestconfig: pytest.Config) -> types.ProxyConfig: + """The environment proxy to pass on to the charm/testing model.""" + proxy = pytestconfig.getoption("--proxy") + no_proxy = pytestconfig.getoption("--no-proxy") + return types.ProxyConfig(http=proxy, https=proxy, no_proxy=no_proxy) + + @pytest_asyncio.fixture(scope="module", name="ssh_connection") async def ssh_connection_fixture( openstack_server: Server, openstack_metadata: OpenstackMeta, proxy: types.ProxyConfig From 4ec345f66fcee9a5c5b1659de849b0d2da26b0b8 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 08:55:22 +0000 Subject: [PATCH 49/63] catch tiemout for retry --- tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index c96033f..d9af844 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -265,7 +265,7 @@ def wait_for_valid_connection( # pylint: disable=too-many-arguments if result.ok: _install_proxy(conn=ssh_connection, proxy=proxy) return ssh_connection - except NoValidConnectionsError as exc: + except (NoValidConnectionsError, TimeoutError) as exc: logger.warning("Connection not yet ready, %s.", str(exc)) time.sleep(10) raise TimeoutError("No valid ssh connections found.") From 12ee04ab70d7144ee983631caed94532f8d464d3 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 11:46:37 +0000 Subject: [PATCH 50/63] openstack image upload arch --- src/github_runner_image_builder/cli.py | 1 + src/github_runner_image_builder/config.py | 13 +++++++++++++ src/github_runner_image_builder/store.py | 7 ++++++- tests/integration/conftest.py | 4 ++-- tests/unit/test_config.py | 16 ++++++++++++++++ tests/unit/test_store.py | 2 ++ 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index 6562868..f98ed6f 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -198,6 +198,7 @@ def _build_and_upload( base_image = BaseImage.from_str(base) builder.build_image(arch=arch, base_image=base_image) image_id = store.upload_image( + arch=arch, cloud_name=cloud_name, image_name=image_name, image_path=IMAGE_OUTPUT_PATH, diff --git a/src/github_runner_image_builder/config.py b/src/github_runner_image_builder/config.py index 0f91e77..5f361ef 100644 --- a/src/github_runner_image_builder/config.py +++ b/src/github_runner_image_builder/config.py @@ -52,6 +52,19 @@ class Arch(str, Enum): ARM64 = "arm64" X64 = "x64" + def to_openstack(self) -> str: + """Convert the architecture to OpenStack compatible arch string. + + Returns: + The architecture string. + """ # noqa: DCO050 the ValueError is an unreachable code. + match self: + case Arch.ARM64: + return "aarch64" + case Arch.X64: + return "x86_64" + raise ValueError # pragma: nocover + ARCHITECTURES_ARM64 = {"aarch64", "arm64"} ARCHITECTURES_X86 = {"x86_64"} diff --git a/src/github_runner_image_builder/store.py b/src/github_runner_image_builder/store.py index e7bd7b8..aaeffc9 100644 --- a/src/github_runner_image_builder/store.py +++ b/src/github_runner_image_builder/store.py @@ -12,15 +12,19 @@ import openstack.exceptions from openstack.image.v2.image import Image +from github_runner_image_builder.config import Arch from github_runner_image_builder.errors import OpenstackError, UploadImageError logger = logging.getLogger(__name__) -def upload_image(cloud_name: str, image_name: str, image_path: Path, keep_revisions: int) -> str: +def upload_image( + arch: Arch, cloud_name: str, image_name: str, image_path: Path, keep_revisions: int +) -> str: """Upload image to openstack glance. Args: + arch: The image architecture. cloud_name: The Openstack cloud to use from clouds.yaml. image_name: The image name to upload as. image_path: The path to image to upload. @@ -38,6 +42,7 @@ def upload_image(cloud_name: str, image_name: str, image_path: Path, keep_revisi name=image_name, filename=str(image_path), allow_duplicates=True, + properties={"architecture": arch.to_openstack()}, wait=True, ) _prune_old_images( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5a217b3..c1a333c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -202,10 +202,10 @@ def openstack_metadata_fixture( @pytest.fixture(scope="module", name="openstack_security_group") def openstack_security_group_fixture(openstack_connection: Connection): """An ssh-connectable security group.""" - security_group_name = "github-runner-image-builder-operator-test-security-group" + security_group_name = "github-runner-image-builder-test-security-group" security_group: SecurityGroup = openstack_connection.create_security_group( name=security_group_name, - description="For servers managed by the github-runner-image-builder charm.", + description="For servers managed by the github-runner-image-builder app.", ) # For ping openstack_connection.create_security_group_rule( diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 40958fd..00de8c2 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -18,6 +18,22 @@ ) +@pytest.mark.parametrize( + "arch, expected", + [ + pytest.param(Arch.ARM64, "aarch64", id="arm64"), + pytest.param(Arch.X64, "x86_64", id="amd64"), + ], +) +def test_arch_openstack_conversion(arch: Arch, expected: str): + """ + arrange: given platform architecture. + act: when arch.to_openstack is called. + assert: expected Openstack architecture is returned. + """ + assert arch.to_openstack() == expected + + @pytest.mark.parametrize( "arch", [ diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index 8c5e230..c1a4708 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -127,6 +127,7 @@ def test_upload_image_error(mock_connection: MagicMock): with pytest.raises(UploadImageError) as exc: store.upload_image( + arch=MagicMock(), cloud_name=MagicMock(), image_name=MagicMock(), image_path=MagicMock(), @@ -146,6 +147,7 @@ def test_upload_image(mock_connection: MagicMock): assert ( store.upload_image( + arch=MagicMock(), cloud_name=MagicMock(), image_name=MagicMock(), image_path=MagicMock(), From 22e079a2ea5db482cb8dee024a19970ca85320a1 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 13:31:39 +0000 Subject: [PATCH 51/63] increase ssh timeout --- tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index d9af844..2bc92b4 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -258,7 +258,7 @@ def wait_for_valid_connection( # pylint: disable=too-many-arguments host=ip, user="ubuntu", connect_kwargs={"key_filename": str(ssh_key.absolute())}, - connect_timeout=10, + connect_timeout=10 * 60, ) try: result: Result = ssh_connection.run("echo 'hello world'") From cd671460fed6fd1854736db50e94980a3aa6f015 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Wed, 12 Jun 2024 16:09:08 +0000 Subject: [PATCH 52/63] add server info logging on exception --- tests/integration/conftest.py | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c1a333c..ac5bf9b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,6 +9,7 @@ from pathlib import Path import openstack +import openstack.exceptions import pytest import pytest_asyncio import yaml @@ -249,24 +250,27 @@ async def openstack_server_fixture( server_name = f"test-server-{test_id}" images: list[Image] = openstack_metadata.connection.search_images(openstack_image_name) assert images, "No built image found." - server: Server = openstack_metadata.connection.create_server( - name=server_name, - image=images[0], - key_name=openstack_metadata.ssh_key.keypair.name, - auto_ip=False, - # these are pre-configured values on private endpoint. - security_groups=[openstack_security_group.name], - flavor=openstack_metadata.flavor, - network=openstack_metadata.network, - timeout=120, - wait=True, - ) - - yield server - - openstack_metadata.connection.delete_server(server_name, wait=True) - for image in images: - openstack_metadata.connection.delete_image(image.id) + try: + server: Server = openstack_metadata.connection.create_server( + name=server_name, + image=images[0], + key_name=openstack_metadata.ssh_key.keypair.name, + auto_ip=False, + # these are pre-configured values on private endpoint. + security_groups=[openstack_security_group.name], + flavor=openstack_metadata.flavor, + network=openstack_metadata.network, + timeout=120, + wait=True, + ) + yield server + except openstack.exceptions.SDKException: + server = openstack_metadata.connection.get_server(name_or_id=server_name) + logger.exception("Failed to create server, %s", dict(server)) + finally: + openstack_metadata.connection.delete_server(server_name, wait=True) + for image in images: + openstack_metadata.connection.delete_image(image.id) @pytest.fixture(scope="module", name="proxy") From 93c346193d6d4e35d0084cf5fba304b88a493913 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 13 Jun 2024 01:46:00 +0000 Subject: [PATCH 53/63] debug --- .github/workflows/integration_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index a40d51c..e412c94 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -26,7 +26,9 @@ jobs: # need to run in sudo mode due to chroot - name: Run integration tests run: sudo $(which tox) -e integration -- -m arm64 --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS }} - + - name: Tmate + if: ${{ failure() }} + uses: canonical/action-tmate@main integration-tests-amd: name: Integration test (X64) runs-on: [self-hosted, X64, jammy, stg-private-endpoint] From 96000db0ce705dfa1a1aa3c09efd1c861c5c9cc2 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 13 Jun 2024 07:16:44 +0000 Subject: [PATCH 54/63] callback script abs pathing --- src/github_runner_image_builder/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github_runner_image_builder/cli.py b/src/github_runner_image_builder/cli.py index f98ed6f..e91585a 100644 --- a/src/github_runner_image_builder/cli.py +++ b/src/github_runner_image_builder/cli.py @@ -206,4 +206,4 @@ def _build_and_upload( ) if callback_script_path: # The callback script is a user trusted script. - subprocess.check_call([f"./{callback_script_path}", image_id]) # nosec: B603 + subprocess.check_call([str(callback_script_path), image_id]) # nosec: B603 From 36e6aeba312153d8c51611fa5ac358265d6a53ec Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 13 Jun 2024 07:17:39 +0000 Subject: [PATCH 55/63] callback script abs pathing --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ac5bf9b..76a2125 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -318,7 +318,7 @@ def cli_run_fixture( "--keep-revisions", "2", "--callback-script", - str(callback_script), + str(callback_script.absolute()), ] ) From 122ee5f06b2502bf37f594273b2f8cb87f0b8341 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Thu, 13 Jun 2024 14:38:11 +0000 Subject: [PATCH 56/63] pin versions --- .github/workflows/integration_test.yaml | 4 ++-- tests/integration/requirements.txt | 6 +++--- tests/unit/requirements.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index e412c94..ac10414 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -15,7 +15,7 @@ jobs: matrix: image: [jammy, noble] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - uses: canonical/setup-lxd@v0.1.1 - name: Install tox run: | @@ -36,7 +36,7 @@ jobs: matrix: image: [jammy, noble] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - uses: canonical/setup-lxd@v0.1.1 - name: Install tox run: | diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 16911e9..461399f 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,3 +1,3 @@ -pylxd -fabric -types-paramiko +pylxd==2.3.4 +fabric==3.2.2 +types-paramiko==3.4.0.20240423 diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt index c9c3f26..a20f152 100644 --- a/tests/unit/requirements.txt +++ b/tests/unit/requirements.txt @@ -1 +1 @@ -factory-boy>=3,<4 +factory-boy==3.3.0 From 827471d3f24f61c163d9e106aeaecf12ce11bed7 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 14 Jun 2024 02:52:38 +0000 Subject: [PATCH 57/63] reload shell --- tests/integration/test_image.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index 71de102..5d2f142 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -26,10 +26,12 @@ class Commands(NamedTuple): Attributes: name: The test name. command: The command to execute. + reload: Whether the shell should be reloaded. """ name: str command: str + reload: bool | None # This is matched with E2E test run of github-runner-operator charm. @@ -59,7 +61,7 @@ class Commands(NamedTuple): Commands(name="yq version", command="yq --version"), Commands(name="apt update", command="sudo apt-get update -y"), Commands(name="install pipx", command="sudo apt-get install -y pipx"), - Commands(name="pipx add path", command="pipx ensurepath"), + Commands(name="pipx add path", command="pipx ensurepath", reload=True), Commands(name="install check-jsonschema", command="pipx install check-jsonschema"), Commands(name="check jsonschema", command="check-jsonschema --version"), Commands(name="unzip version", command="unzip -v"), @@ -121,6 +123,10 @@ async def test_image_arm(ssh_connection: SSHConnection): logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) + if testcmd.reload: + logger.info("Reloading connection after command: %s", testcmd.command) + ssh_connection.close() + ssh_connection.open() assert result.return_code == 0 From 1232bcf7980b823f12555c7e346df0d6aebf438d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 14 Jun 2024 02:57:36 +0000 Subject: [PATCH 58/63] lint fix --- tests/integration/test_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index 5d2f142..05830e5 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -31,7 +31,7 @@ class Commands(NamedTuple): name: str command: str - reload: bool | None + reload: bool | None = None # This is matched with E2E test run of github-runner-operator charm. From 7c223ab6d10ecf75331a7054f85f2cddc89e4f32 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Fri, 14 Jun 2024 08:34:52 +0000 Subject: [PATCH 59/63] add path --- tests/integration/test_image.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_image.py b/tests/integration/test_image.py index 05830e5..ca0844a 100644 --- a/tests/integration/test_image.py +++ b/tests/integration/test_image.py @@ -26,12 +26,12 @@ class Commands(NamedTuple): Attributes: name: The test name. command: The command to execute. - reload: Whether the shell should be reloaded. + env: Additional run envs. """ name: str command: str - reload: bool | None = None + env: dict | None = None # This is matched with E2E test run of github-runner-operator charm. @@ -61,9 +61,15 @@ class Commands(NamedTuple): Commands(name="yq version", command="yq --version"), Commands(name="apt update", command="sudo apt-get update -y"), Commands(name="install pipx", command="sudo apt-get install -y pipx"), - Commands(name="pipx add path", command="pipx ensurepath", reload=True), + Commands(name="pipx add path", command="pipx ensurepath"), Commands(name="install check-jsonschema", command="pipx install check-jsonschema"), - Commands(name="check jsonschema", command="check-jsonschema --version"), + Commands( + name="check jsonschema", + command="check-jsonschema --version", + # pipx has been added to PATH but still requires additional PATH env since + # default shell is not bash in OpenStack + env={"PATH": "$PATH:/home/ubuntu/.local/bin"}, + ), Commands(name="unzip version", command="unzip -v"), Commands(name="gh version", command="gh --version"), Commands( @@ -121,12 +127,8 @@ async def test_image_arm(ssh_connection: SSHConnection): """ for testcmd in TEST_RUNNER_COMMANDS: logger.info("Running command: %s", testcmd.command) - result: Result = ssh_connection.run(testcmd.command) + result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - if testcmd.reload: - logger.info("Reloading connection after command: %s", testcmd.command) - ssh_connection.close() - ssh_connection.open() assert result.return_code == 0 From 6721de1025b4e8bbba7a94cdca93dfaa3a03ab6d Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 17 Jun 2024 03:27:46 +0000 Subject: [PATCH 60/63] test: increase server create timeout --- tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 76a2125..b696c90 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -260,7 +260,7 @@ async def openstack_server_fixture( security_groups=[openstack_security_group.name], flavor=openstack_metadata.flavor, network=openstack_metadata.network, - timeout=120, + timeout=60 * 20, wait=True, ) yield server From c9f7d90383bec35f8be42db7b3fc685b36b26078 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 17 Jun 2024 04:38:21 +0000 Subject: [PATCH 61/63] test: wait for snapd --- tests/integration/helpers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 2bc92b4..f758375 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -280,6 +280,8 @@ def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): """ if not proxy or not proxy.http: return + wait_for(partial(_snap_ready, conn)) + command = "sudo snap install aproxy --edge" logger.info("Running command: %s", command) result: Result = conn.run(command) @@ -312,3 +314,18 @@ def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): logger.info("Running command: %s", command) result = conn.run(command) assert result.ok, "Failed to configure iptable rules" + + +def _snap_ready(conn: SSHConnection) -> bool: + """Checks whether snapd is ready. + + Args: + conn: The SSH connection instance. + + Returns: + Whether snapd is ready. + """ + command = "sudo systemctl is-active snapd.seeded.service" + logger.info("Running command: %s", command) + result: Result = conn.run(command) + return result.ok From 539ba32a3dc069eda3ff62c93d56e07e8fd5b726 Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 17 Jun 2024 04:47:26 +0000 Subject: [PATCH 62/63] test: async wait for --- tests/integration/conftest.py | 2 +- tests/integration/helpers.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b696c90..f0f7fd5 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -287,7 +287,7 @@ async def ssh_connection_fixture( ) -> SSHConnection: """The openstack server ssh connection fixture.""" logger.info("Setting up SSH connection.") - ssh_connection = helpers.wait_for_valid_connection( + ssh_connection = await helpers.wait_for_valid_connection( connection=openstack_metadata.connection, server_name=openstack_server.name, network=openstack_metadata.network, diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f758375..68038f0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -221,7 +221,7 @@ def _instance_running(instance: Instance) -> bool: # All the arguments are necessary -def wait_for_valid_connection( # pylint: disable=too-many-arguments +async def wait_for_valid_connection( # pylint: disable=too-many-arguments connection: Connection, server_name: str, network: str, @@ -263,7 +263,7 @@ def wait_for_valid_connection( # pylint: disable=too-many-arguments try: result: Result = ssh_connection.run("echo 'hello world'") if result.ok: - _install_proxy(conn=ssh_connection, proxy=proxy) + await _install_proxy(conn=ssh_connection, proxy=proxy) return ssh_connection except (NoValidConnectionsError, TimeoutError) as exc: logger.warning("Connection not yet ready, %s.", str(exc)) @@ -271,7 +271,7 @@ def wait_for_valid_connection( # pylint: disable=too-many-arguments raise TimeoutError("No valid ssh connections found.") -def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): +async def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): """Run commands to install proxy. Args: @@ -280,7 +280,7 @@ def _install_proxy(conn: SSHConnection, proxy: types.ProxyConfig | None = None): """ if not proxy or not proxy.http: return - wait_for(partial(_snap_ready, conn)) + await wait_for(partial(_snap_ready, conn)) command = "sudo snap install aproxy --edge" logger.info("Running command: %s", command) From 59ce40d6b5857e17011bd3522ff954c11489b0bb Mon Sep 17 00:00:00 2001 From: Yanks Yoon Date: Mon, 17 Jun 2024 05:53:30 +0000 Subject: [PATCH 63/63] test: unexpected exit code --- tests/integration/helpers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 68038f0..3f90423 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -16,6 +16,7 @@ from fabric import Connection as SSHConnection from fabric import Result +from invoke.exceptions import UnexpectedExit from openstack.compute.v2.server import Server from openstack.connection import Connection from paramiko.ssh_exception import NoValidConnectionsError @@ -327,5 +328,8 @@ def _snap_ready(conn: SSHConnection) -> bool: """ command = "sudo systemctl is-active snapd.seeded.service" logger.info("Running command: %s", command) - result: Result = conn.run(command) - return result.ok + try: + result: Result = conn.run(command) + return result.ok + except UnexpectedExit: + return False