From 35666d79cf565da3be71645013658ef984c020a9 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Mon, 5 Feb 2024 17:58:54 +0100 Subject: [PATCH 01/14] feat: add script for building rocks via lpci --- .gitignore | 3 +- rockcraft_lpci_build/__init__.py | 0 rockcraft_lpci_build/requirements.sh | 4 + rockcraft_lpci_build/requirements.txt | 4 + rockcraft_lpci_build/rockcraft_lpci_build.py | 471 +++++++++++++++++++ 5 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 rockcraft_lpci_build/__init__.py create mode 100755 rockcraft_lpci_build/requirements.sh create mode 100644 rockcraft_lpci_build/requirements.txt create mode 100755 rockcraft_lpci_build/rockcraft_lpci_build.py diff --git a/.gitignore b/.gitignore index 600d2d3..9f94e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.vscode \ No newline at end of file +.vscode +__pycache__ diff --git a/rockcraft_lpci_build/__init__.py b/rockcraft_lpci_build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rockcraft_lpci_build/requirements.sh b/rockcraft_lpci_build/requirements.sh new file mode 100755 index 0000000..6cc28eb --- /dev/null +++ b/rockcraft_lpci_build/requirements.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +sudo apt update +sudo apt install -y distro-info \ No newline at end of file diff --git a/rockcraft_lpci_build/requirements.txt b/rockcraft_lpci_build/requirements.txt new file mode 100644 index 0000000..c5b5ebb --- /dev/null +++ b/rockcraft_lpci_build/requirements.txt @@ -0,0 +1,4 @@ +distro-info +GitPython +launchpadlib +pyyaml \ No newline at end of file diff --git a/rockcraft_lpci_build/rockcraft_lpci_build.py b/rockcraft_lpci_build/rockcraft_lpci_build.py new file mode 100755 index 0000000..dc5d320 --- /dev/null +++ b/rockcraft_lpci_build/rockcraft_lpci_build.py @@ -0,0 +1,471 @@ +#!/usr/bin/python3 + +import argparse +import atexit +import base64 +import distro_info +import logging +import os +import shutil +import sys +import tempfile +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +import requests + +import yaml +from git import Repo + +# Launchpad API docs: https://launchpad.net/+apidoc/devel.html +from launchpadlib.launchpad import Launchpad +from lazr.restfulclient.resource import Entry + +# lpci reference: https://lpci.readthedocs.io/en/latest/configuration.html +LPCI_CONFIG_TEMPLATE = """ +pipeline: +- build-rock + +jobs: + build-rock: + # The "series" field is included by the code + # The "architectures" field is included by the code + snaps: + # - name: lxd + - name: chisel + channel: latest/candidate + - name: rockcraft + classic: true + run: | + # lxd waitready + # lxd init --auto + # snap set lxd daemon.group=adm + # snap restart lxd + HTTPS_PROXY=${https_proxy} HTTP_PROXY=${http_proxy} rockcraft pack \ + --verbosity=trace --destructive-mode + output: + paths: + - "*.rock" +""" + + +class LaunchpadBuildTimeout(Exception): + pass + + +class LaunchpadBuildFailure(Exception): + pass + + +class RockcraftLpciBuilds: + def __init__(self) -> None: + logging.basicConfig(level=logging.INFO) + + self.args = self.cli_args().parse_args() + self.set_lp_creds() + self.app_name = "rockcraft-lpci" + self.rockcraft_yaml = Path("rockcraft.yaml") + self.rockcraft_yaml_raw = self.read_rockcraft_yaml() + try: + self.rock_name = self.rockcraft_yaml_raw["name"] + except KeyError: + logging.exception(f"{self.rockcraft_yaml} is missing the 'name' field") + raise + self.launchpad = self.lp_login("production") + self.lp_user = self.launchpad.me.name + self.lp_owner = f"/~{self.lp_user}" + self.lp_repo_name = f"{self.app_name}-{self.rock_name}-{int(time.time())}" + self.lp_repo_path = f"~{self.lp_user}/+git/{self.lp_repo_name}" + + @staticmethod + def cli_args() -> argparse.ArgumentParser: + """Arguments parser""" + parser = argparse.ArgumentParser( + description="Builds rocks in Launchpad, with lpci.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + lp_creds = parser.add_mutually_exclusive_group(required=True) + # E.g. if the LP credential file looks like: + # [1] + # consumer_key = System-wide: Debian GNU/Linux (9df369915b99) + # consumer_secret = + # access_token = foo + # access_secret = bar + # then this arg's value should be `cat | base64 -w 0` + lp_creds.add_argument( + "--lp-credentials-b64", + help="raw, single-line and base64 enconded Launchpad credentials", + ) + lp_creds.add_argument( + "--lp-credentials-file", + help=str( + "the path to an existing Launchpad credentials file." + "If passed, --lp-credentials-b64 is ignored" + ), + ) + parser.add_argument( + "--timeout", + default=3600, + type=int, + help=str( + "time (in sec) after which to stop waiting for the build to finish" + ), + ) + parser.add_argument( + "--allow-build-failures", + action="store_true", + help=str("acknowledge that uploaded project will be publicly available"), + ) + parser.add_argument( + "--launchpad-accept-public-upload", + action="store_true", + help=str("for multi-arch builds, continue even if some builds fail"), + ) + + return parser + + @staticmethod + def delete_file(file_path: str) -> None: + """Delete file""" + try: + os.remove(file_path) + logging.info(f"File {file_path} deleted successfully.") + except OSError as e: + logging.exception(f"Error deleting file {file_path}: {e}") + + @staticmethod + def lp_login_failure() -> None: + """Callback function for when the Launchpad login fails""" + logging.error("Unable to login to Launchpad with the provided credentials") + sys.exit(1) + + @staticmethod + def delete_git_repository(lp_client: Launchpad, lp_repo_path: str) -> None: + git_repo = lp_client.git_repositories.getByPath(path=lp_repo_path) # type: ignore + + if git_repo is None: + return + + logging.info(f"Deleting repository {lp_repo_path} from Launchpad...") + git_repo.lp_delete() + + @staticmethod + def save_build_logs(lp_build: Entry) -> dict: + ci_build = requests.get(lp_build.ci_build_link) + ci_build.raise_for_status() + ci_build = ci_build.json() + + if "build_log_url" in ci_build and ci_build["build_log_url"]: + ci_build_logs = requests.get(ci_build["build_log_url"]) + with tempfile.NamedTemporaryFile(delete=False) as log: + logging.info(f"Build log save at {log.name}") + log.write(ci_build_logs.text.encode()) + + else: + logging.warning( + f"Unable to get logs. build_log_url not in {lp_build.ci_build_link}." + ) + + return ci_build + + def ack_project_will_be_public(self) -> None: + if self.args.launchpad_accept_public_upload: + return + + print( + "Your current directory will be sent to Launchpad and will be public!\n" + "Are you sure you want to continue? [press y to continue]: " + ) + key = input() + if key != "y": + sys.exit(0) + + def read_rockcraft_yaml(self) -> dict: + self.check_rockcraft_yaml() + with open(self.rockcraft_yaml) as rockfile: + try: + return yaml.safe_load(rockfile) + except yaml.scanner.ScannerError: + logging.exception(f"{self.rockcraft_yaml} cannot be read") + raise + + def set_lp_creds(self) -> None: + if self.args.lp_credentials_file: + self.lp_creds = self.args.lp_credentials_file + logging.info(f"Using file '{self.lp_creds}' for Launchpad authentication") + else: + fd, self.lp_creds = tempfile.mkstemp() + atexit.register(self.delete_file, self.lp_creds) + + with os.fdopen(fd, "w") as tmp_lp_creds: + tmp_lp_creds.write( + base64.b64decode(self.args.lp_credentials_b64).decode() + ) + + logging.info(f"Saved Launchpad credentials in {self.lp_creds}") + + def lp_login(self, lp_server: str) -> Launchpad: + """Login to Launchpad""" + return Launchpad.login_with( + f"{self.rock_name} remote-build", + lp_server, + credentials_file=self.lp_creds, + credential_save_failed=self.lp_login_failure, + version="devel", + ) + + def check_rockcraft_yaml(self) -> None: + if not self.rockcraft_yaml.exists(): + raise FileNotFoundError(f"File {self.rockcraft_yaml} not found") + + def create_git_repository(self) -> Entry: + """Create git repository in LP""" + logging.info( + "Creating git repo: name=%s, owner=%s, target=%s", + self.lp_repo_name, + self.lp_owner, + self.lp_owner, + ) + return self.launchpad.git_repositories.new( + name=self.lp_repo_name, owner=self.lp_owner, target=self.lp_owner + ) + + def prepare_local_project(self) -> None: + self.lp_local_repo_path = tempfile.mkdtemp() + project_path = os.getcwd() + logging.info( + f"Copying project from {project_path} to {self.lp_local_repo_path}" + ) + shutil.copytree(project_path, self.lp_local_repo_path, dirs_exist_ok=True) + + logging.info(f"Initializing a new Git repo at {self.lp_local_repo_path}") + if Path(f"{self.lp_local_repo_path}/.git").exists(): + shutil.rmtree(f"{self.lp_local_repo_path}/.git") + + # Just making sure we don't push the lp credentials + if Path( + f"{self.lp_local_repo_path}/{os.path.basename(self.lp_creds)}" + ).exists(): + os.remove(f"{self.lp_local_repo_path}/{os.path.basename(self.lp_creds)}") + + self.lp_local_repo = Repo.init(self.lp_local_repo_path) + + def get_rock_archs(self) -> list: + try: + platforms = self.rockcraft_yaml_raw["platforms"] + except KeyError: + logging.exception(f"{self.rockcraft_yaml} is missing the platforms") + raise + + archs = [] + for platf, values in platforms.items(): + if isinstance(values, dict) and "build-for" in values: + archs.append(values["build-for"]) + continue + + archs.append(platf) + + return list(set(archs)) + + def get_rock_build_base(self) -> str: + try: + build_base = self.rockcraft_yaml_raw["build_base"] + except KeyError: + try: + build_base = self.rockcraft_yaml_raw["base"] + except KeyError: + logging.exception(f"{self.rockcraft_yaml} is missing the 'base' field") + raise + + if build_base == "devel": + return distro_info.UbuntuDistroInfo().devel() + + all_releases, all_codenames = ( + distro_info.UbuntuDistroInfo().get_all(result="fullname"), + distro_info.UbuntuDistroInfo().get_all(), + ) + + build_base_release = build_base.replace(":", "@").split("@")[-1] + build_base_full_release = list( + filter(lambda r: build_base_release in r, all_releases) + )[0] + + return all_codenames[all_releases.index(build_base_full_release)] + + def write_lpci_configuration_file(self) -> None: + lpci_config = yaml.safe_load(LPCI_CONFIG_TEMPLATE) + archs = self.get_rock_archs() + build_base = self.get_rock_build_base() + + logging.info( + f" !! This rock ({self.rock_name}) is being built on " + f"{build_base}, for: {archs} !!" + ) + self.target_build_count = len(archs) + lpci_config["jobs"]["build-rock"]["architectures"] = archs + lpci_config["jobs"]["build-rock"]["series"] = build_base + lpci_config_file = f"{self.lp_local_repo_path}/.launchpad.yaml" + logging.info(f"LPCI configuration file saved in {lpci_config_file}") + + with open(f"{self.lp_local_repo_path}/.launchpad.yaml", "w") as lpci_file: + yaml.dump(lpci_config, lpci_file) + + def get_lp_token(self) -> str: + # Add an extra 5min to the token just to make sure this script exits + # before the token expires. + date_expires = datetime.now(timezone.utc) + timedelta( + seconds=self.args.timeout + 300 + ) + logging.info( + f"Creating new Launchpad token for {self.lp_repo_name}. " + f"It will expire on {date_expires.strftime('%Y-%m-%dT%H:%M:%S %Z')}" + ) + return self.lp_repo.issueAccessToken( # type: ignore + description=f"rockcraft remote-build for {self.rock_name}", + scopes=["repository:push"], + date_expires=date_expires.isoformat(), + ) + + def push_to_lp(self, repo_url: str) -> None: + self.lp_local_repo.git.add(A=True) + self.lp_local_repo.index.commit(f"Initial commit: build {self.rock_name}") + + # Create a new branch + branch_name = "master" + # self.lp_local_repo.git.branch(branch_name) + self.lp_local_repo.git.checkout(branch_name) + + logging.info( + f"Pushing local project {self.lp_local_repo_path} " + f"to {self.lp_repo.git_https_url}" + ) + origin = self.lp_local_repo.create_remote("origin", url=repo_url) + origin.push(f"{branch_name}:{branch_name}") + + def wait_for_lp_builds(self) -> list: + logging.info( + f"Waiting for builds to finish at {self.lp_repo_path}, " + f"on branch {self.lp_local_repo.active_branch.name}" + ) + + keep_waiting = True + wait_until = datetime.now() + timedelta(seconds=self.args.timeout) + finished_builds = [] + successful_builds = [] + while keep_waiting: + if wait_until < datetime.now(): + logging.error("Timed out. Keeping the Launchpad repo alive") + atexit.unregister(self.delete_git_repository) + raise LaunchpadBuildTimeout + + build_status = self.lp_repo.getStatusReports( + commit_sha1=self.lp_local_repo.head.commit.hexsha + ) + if len(build_status) != self.target_build_count: + logging.warning( + f"Need {self.target_build_count} builds " + f"but Launchpad only listed {len(build_status)} so far. Waiting" + ) + time.sleep(5) + continue + + for build in build_status: + if build.ci_build_link in finished_builds: + logging.debug(f"{build.ci_build_link} has finished already") + continue + + logging.debug(f"Tracking build at {build.ci_build_link}") + if build.result in ["Failed", "Skipped", "Cancelled", "Succeeded"]: + finished_builds.append(build.ci_build_link) + ci_build = self.save_build_logs(build) + log_msg_prefix = f"[{ci_build.get('arch_tag', 'unknown arch')}]" + if build.result == "Succeeded": + logging.info(f"{log_msg_prefix} Build successful!") + successful_builds.append(build) + continue + + # If it gets here, it means it is finished and not successful + error_msg = f"{log_msg_prefix} Build failed!" + if self.args.allow_build_failures: + logging.error(f"{error_msg}. Continuing") + continue + else: + logging.error(f"{error_msg}. Keeping the Launchpad repo alive") + atexit.unregister(self.delete_git_repository) + raise LaunchpadBuildFailure() + + # If we got here, it means the build is still in progress + # We'll keep going until len(finished_builds) >= len(build_status) + if len(finished_builds) >= len(build_status): + logging.info("All builds have finished") + break + + logging.info( + f"{len(finished_builds)}/{len(build_status)} builds finished, waiting" + ) + time.sleep(30) + + return successful_builds + + def download_build_artefacts(successful_builds: list) -> None: + for build in successful_builds: + artefact_urls = build.getArtifactURLs() + rock_url = list(filter(lambda u: ".rock" in u, artefact_urls)) + if not rock_url: + arch = build.distro_arch_series_link.split("/")[-1] + logging.warning( + f"No rock artefacts found for {arch} (job {build.title})" + ) + continue + for url in rock_url: + r = requests.get(url) + r.raise_for_status() + + out_file = url.split("/")[-1] + with open(out_file, "wb") as oci_archive: + oci_archive.write(r.content) + + logging.info(f"Downloaded {out_file} into current directory") + + def run(self) -> None: + """Main function""" + self.ack_project_will_be_public() + logging.info(f"[launchpad] Logged in as {self.lp_user} ({self.launchpad.me})") + self.prepare_local_project() + + logging.info(f"Creating .launchpad.yaml file...") + self.write_lpci_configuration_file() + self.lp_repo = self.create_git_repository() + atexit.register(self.delete_git_repository, self.launchpad, self.lp_repo_path) + token = self.get_lp_token() + lp_repo_url = ( + f"https://{self.lp_user}:{token}@git.launchpad.net/" + f"~{self.lp_user}/+git/{self.lp_repo_name}/" + ) + logging.info( + f"The remote for {self.lp_repo_name} is {lp_repo_url.replace(token, '***')}" + ) + try: + self.push_to_lp(lp_repo_url) + except: + logging.exception("Failed to push local project to Launchpad") + # Graceful termination to allow for the cleanup + return + + logging.info( + " !! You can follow your builds at " + f"{self.lp_repo.web_link}/+ref/{self.lp_local_repo.active_branch.name} !!" + ) + + successful_builds = self.wait_for_lp_builds() + + if not successful_builds: + logging.error(f"No builds were successful! There are no rocks to retrieve") + return + + self.download_build_artefacts(successful_builds) + + +if __name__ == "__main__": + builder = RockcraftLpciBuilds() + builder.run() From fd67f09329d7a8cb2c3e02bd97d1519d1dd470b4 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Mon, 5 Feb 2024 18:37:05 +0100 Subject: [PATCH 02/14] fix: linting --- rockcraft_lpci_build/rockcraft_lpci_build.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rockcraft_lpci_build/rockcraft_lpci_build.py b/rockcraft_lpci_build/rockcraft_lpci_build.py index dc5d320..b0a46bf 100755 --- a/rockcraft_lpci_build/rockcraft_lpci_build.py +++ b/rockcraft_lpci_build/rockcraft_lpci_build.py @@ -3,7 +3,6 @@ import argparse import atexit import base64 -import distro_info import logging import os import shutil @@ -12,8 +11,9 @@ import time from datetime import datetime, timedelta, timezone from pathlib import Path -import requests +import distro_info +import requests import yaml from git import Repo @@ -28,7 +28,7 @@ jobs: build-rock: - # The "series" field is included by the code + # The "series" field is included by the code # The "architectures" field is included by the code snaps: # - name: lxd @@ -433,7 +433,7 @@ def run(self) -> None: logging.info(f"[launchpad] Logged in as {self.lp_user} ({self.launchpad.me})") self.prepare_local_project() - logging.info(f"Creating .launchpad.yaml file...") + logging.info("Creating .launchpad.yaml file...") self.write_lpci_configuration_file() self.lp_repo = self.create_git_repository() atexit.register(self.delete_git_repository, self.launchpad, self.lp_repo_path) @@ -447,9 +447,9 @@ def run(self) -> None: ) try: self.push_to_lp(lp_repo_url) - except: + except Exception: + # Catch anything, for a graceful termination, to allow for the cleanup logging.exception("Failed to push local project to Launchpad") - # Graceful termination to allow for the cleanup return logging.info( @@ -460,7 +460,7 @@ def run(self) -> None: successful_builds = self.wait_for_lp_builds() if not successful_builds: - logging.error(f"No builds were successful! There are no rocks to retrieve") + logging.error("No builds were successful! There are no rocks to retrieve") return self.download_build_artefacts(successful_builds) From ea6f8bd17d092a101482b9cbde83afedd4700f98 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Mon, 5 Feb 2024 19:06:53 +0100 Subject: [PATCH 03/14] fix: docstrings and other pylint warnings --- rockcraft_lpci_build/rockcraft_lpci_build.py | 166 +++++++++++-------- 1 file changed, 101 insertions(+), 65 deletions(-) diff --git a/rockcraft_lpci_build/rockcraft_lpci_build.py b/rockcraft_lpci_build/rockcraft_lpci_build.py index b0a46bf..e98a202 100755 --- a/rockcraft_lpci_build/rockcraft_lpci_build.py +++ b/rockcraft_lpci_build/rockcraft_lpci_build.py @@ -1,5 +1,8 @@ #!/usr/bin/python3 +"""Takes a rockcraft.yaml file from the current directory and offloads the +corresponding builds to Launchpad, via lpci.""" + import argparse import atexit import base64 @@ -50,14 +53,16 @@ class LaunchpadBuildTimeout(Exception): - pass + """Custom exception for LP timeouts""" class LaunchpadBuildFailure(Exception): - pass + """Custom exception for LP build failures""" class RockcraftLpciBuilds: + """The LPCI build class""" + def __init__(self) -> None: logging.basicConfig(level=logging.INFO) @@ -69,13 +74,16 @@ def __init__(self) -> None: try: self.rock_name = self.rockcraft_yaml_raw["name"] except KeyError: - logging.exception(f"{self.rockcraft_yaml} is missing the 'name' field") + logging.exception("%s is missing the 'name' field", self.rockcraft_yaml) raise self.launchpad = self.lp_login("production") self.lp_user = self.launchpad.me.name self.lp_owner = f"/~{self.lp_user}" self.lp_repo_name = f"{self.app_name}-{self.rock_name}-{int(time.time())}" self.lp_repo_path = f"~{self.lp_user}/+git/{self.lp_repo_name}" + # The following are defined during the script execution + self.lp_repo = self.lp_local_repo = self.lp_local_repo_path = None + self.target_build_count = 0 @staticmethod def cli_args() -> argparse.ArgumentParser: @@ -129,9 +137,9 @@ def delete_file(file_path: str) -> None: """Delete file""" try: os.remove(file_path) - logging.info(f"File {file_path} deleted successfully.") - except OSError as e: - logging.exception(f"Error deleting file {file_path}: {e}") + logging.info("File %s deleted successfully.", file_path) + except OSError as err: + logging.exception("Error deleting file %s: %s", file_path, err) @staticmethod def lp_login_failure() -> None: @@ -141,16 +149,18 @@ def lp_login_failure() -> None: @staticmethod def delete_git_repository(lp_client: Launchpad, lp_repo_path: str) -> None: + """Delete a git repo from Launchpad""" git_repo = lp_client.git_repositories.getByPath(path=lp_repo_path) # type: ignore if git_repo is None: return - logging.info(f"Deleting repository {lp_repo_path} from Launchpad...") + logging.info("Deleting repository %s from Launchpad...", lp_repo_path) git_repo.lp_delete() @staticmethod def save_build_logs(lp_build: Entry) -> dict: + """Fetch build logs from Launchpad and save them locally""" ci_build = requests.get(lp_build.ci_build_link) ci_build.raise_for_status() ci_build = ci_build.json() @@ -158,17 +168,40 @@ def save_build_logs(lp_build: Entry) -> dict: if "build_log_url" in ci_build and ci_build["build_log_url"]: ci_build_logs = requests.get(ci_build["build_log_url"]) with tempfile.NamedTemporaryFile(delete=False) as log: - logging.info(f"Build log save at {log.name}") + logging.info("Build log save at %s", log.name) log.write(ci_build_logs.text.encode()) else: logging.warning( - f"Unable to get logs. build_log_url not in {lp_build.ci_build_link}." + "Unable to get logs. build_log_url not in %s.", lp_build.ci_build_link ) return ci_build + @staticmethod + def download_build_artefacts(successful_builds: list) -> None: + """Download rocks from the successful LP builds""" + for build in successful_builds: + artefact_urls = build.getArtifactURLs() + rock_url = list(filter(lambda u: ".rock" in u, artefact_urls)) + if not rock_url: + arch = build.distro_arch_series_link.split("/")[-1] + logging.warning( + "No rock artefacts found for %s (job %s)", arch, build.title + ) + continue + for url in rock_url: + download = requests.get(url) + download.raise_for_status() + + out_file = url.split("/")[-1] + with open(out_file, "wb") as oci_archive: + oci_archive.write(download.content) + + logging.info("Downloaded %s into current directory", out_file) + def ack_project_will_be_public(self) -> None: + """Ask for the consent about the project becoming public in Launchpad""" if self.args.launchpad_accept_public_upload: return @@ -181,28 +214,30 @@ def ack_project_will_be_public(self) -> None: sys.exit(0) def read_rockcraft_yaml(self) -> dict: + """Parse the rockcraft.yaml file""" self.check_rockcraft_yaml() - with open(self.rockcraft_yaml) as rockfile: + with open(self.rockcraft_yaml, "r", encoding="utf-8") as rockfile: try: return yaml.safe_load(rockfile) except yaml.scanner.ScannerError: - logging.exception(f"{self.rockcraft_yaml} cannot be read") + logging.exception("%s cannot be read", self.rockcraft_yaml) raise def set_lp_creds(self) -> None: + """Set the LP credentials file locally""" if self.args.lp_credentials_file: self.lp_creds = self.args.lp_credentials_file - logging.info(f"Using file '{self.lp_creds}' for Launchpad authentication") + logging.info("Using file '%s' for Launchpad authentication", self.lp_creds) else: - fd, self.lp_creds = tempfile.mkstemp() + file_d, self.lp_creds = tempfile.mkstemp() atexit.register(self.delete_file, self.lp_creds) - with os.fdopen(fd, "w") as tmp_lp_creds: + with os.fdopen(file_d, "w") as tmp_lp_creds: tmp_lp_creds.write( base64.b64decode(self.args.lp_credentials_b64).decode() ) - logging.info(f"Saved Launchpad credentials in {self.lp_creds}") + logging.info("Saved Launchpad credentials in %s", self.lp_creds) def lp_login(self, lp_server: str) -> Launchpad: """Login to Launchpad""" @@ -215,6 +250,7 @@ def lp_login(self, lp_server: str) -> Launchpad: ) def check_rockcraft_yaml(self) -> None: + """Make sure the rockcraft.yaml file exists""" if not self.rockcraft_yaml.exists(): raise FileNotFoundError(f"File {self.rockcraft_yaml} not found") @@ -231,14 +267,15 @@ def create_git_repository(self) -> Entry: ) def prepare_local_project(self) -> None: + """Initiate a local Git repo for the project""" self.lp_local_repo_path = tempfile.mkdtemp() project_path = os.getcwd() logging.info( - f"Copying project from {project_path} to {self.lp_local_repo_path}" + "Copying project from %s to %s", project_path, self.lp_local_repo_path ) shutil.copytree(project_path, self.lp_local_repo_path, dirs_exist_ok=True) - logging.info(f"Initializing a new Git repo at {self.lp_local_repo_path}") + logging.info("Initializing a new Git repo at %s", self.lp_local_repo_path) if Path(f"{self.lp_local_repo_path}/.git").exists(): shutil.rmtree(f"{self.lp_local_repo_path}/.git") @@ -251,10 +288,11 @@ def prepare_local_project(self) -> None: self.lp_local_repo = Repo.init(self.lp_local_repo_path) def get_rock_archs(self) -> list: + """Infer archs from rockcraft.yaml's platforms""" try: platforms = self.rockcraft_yaml_raw["platforms"] except KeyError: - logging.exception(f"{self.rockcraft_yaml} is missing the platforms") + logging.exception("%s is missing the platforms", self.rockcraft_yaml) raise archs = [] @@ -268,13 +306,14 @@ def get_rock_archs(self) -> list: return list(set(archs)) def get_rock_build_base(self) -> str: + """Infer the Ubuntu series for lpci, from the rockcraft.yaml file""" try: build_base = self.rockcraft_yaml_raw["build_base"] except KeyError: try: build_base = self.rockcraft_yaml_raw["base"] except KeyError: - logging.exception(f"{self.rockcraft_yaml} is missing the 'base' field") + logging.exception("%s is missing the 'base' field", self.rockcraft_yaml) raise if build_base == "devel": @@ -293,32 +332,39 @@ def get_rock_build_base(self) -> str: return all_codenames[all_releases.index(build_base_full_release)] def write_lpci_configuration_file(self) -> None: + """Write the .launchpad.yaml file""" lpci_config = yaml.safe_load(LPCI_CONFIG_TEMPLATE) archs = self.get_rock_archs() build_base = self.get_rock_build_base() logging.info( - f" !! This rock ({self.rock_name}) is being built on " - f"{build_base}, for: {archs} !!" + " !! This rock (%s) is being built on %s, for: %s !!", + self.rock_name, + build_base, + archs, ) self.target_build_count = len(archs) lpci_config["jobs"]["build-rock"]["architectures"] = archs lpci_config["jobs"]["build-rock"]["series"] = build_base lpci_config_file = f"{self.lp_local_repo_path}/.launchpad.yaml" - logging.info(f"LPCI configuration file saved in {lpci_config_file}") + logging.info("LPCI configuration file saved in %s", lpci_config_file) - with open(f"{self.lp_local_repo_path}/.launchpad.yaml", "w") as lpci_file: + with open( + f"{self.lp_local_repo_path}/.launchpad.yaml", "w", encoding="utf-8" + ) as lpci_file: yaml.dump(lpci_config, lpci_file) def get_lp_token(self) -> str: + """Get an LP token for the Git remote URL""" # Add an extra 5min to the token just to make sure this script exits # before the token expires. date_expires = datetime.now(timezone.utc) + timedelta( seconds=self.args.timeout + 300 ) logging.info( - f"Creating new Launchpad token for {self.lp_repo_name}. " - f"It will expire on {date_expires.strftime('%Y-%m-%dT%H:%M:%S %Z')}" + "Creating new Launchpad token for %s. It will expire on %s", + self.lp_repo_name, + date_expires.strftime("%Y-%m-%dT%H:%M:%S %Z"), ) return self.lp_repo.issueAccessToken( # type: ignore description=f"rockcraft remote-build for {self.rock_name}", @@ -327,6 +373,7 @@ def get_lp_token(self) -> str: ) def push_to_lp(self, repo_url: str) -> None: + """Push local git repo to LP""" self.lp_local_repo.git.add(A=True) self.lp_local_repo.index.commit(f"Initial commit: build {self.rock_name}") @@ -336,16 +383,19 @@ def push_to_lp(self, repo_url: str) -> None: self.lp_local_repo.git.checkout(branch_name) logging.info( - f"Pushing local project {self.lp_local_repo_path} " - f"to {self.lp_repo.git_https_url}" + "Pushing local project %s to %s", + self.lp_local_repo_path, + self.lp_repo.git_https_url, ) origin = self.lp_local_repo.create_remote("origin", url=repo_url) origin.push(f"{branch_name}:{branch_name}") def wait_for_lp_builds(self) -> list: + """Wait for all LP builds to finish""" logging.info( - f"Waiting for builds to finish at {self.lp_repo_path}, " - f"on branch {self.lp_local_repo.active_branch.name}" + "Waiting for builds to finish at %s, on branch %s", + self.lp_repo_path, + self.lp_local_repo.active_branch.name, ) keep_waiting = True @@ -363,36 +413,37 @@ def wait_for_lp_builds(self) -> list: ) if len(build_status) != self.target_build_count: logging.warning( - f"Need {self.target_build_count} builds " - f"but Launchpad only listed {len(build_status)} so far. Waiting" + "Need %s builds but Launchpad only listed %s so far. Waiting", + self.target_build_count, + len(build_status), ) time.sleep(5) continue for build in build_status: if build.ci_build_link in finished_builds: - logging.debug(f"{build.ci_build_link} has finished already") + logging.debug("%s has finished already", build.ci_build_link) continue - logging.debug(f"Tracking build at {build.ci_build_link}") + logging.debug("Tracking build at %s", build.ci_build_link) if build.result in ["Failed", "Skipped", "Cancelled", "Succeeded"]: finished_builds.append(build.ci_build_link) ci_build = self.save_build_logs(build) log_msg_prefix = f"[{ci_build.get('arch_tag', 'unknown arch')}]" if build.result == "Succeeded": - logging.info(f"{log_msg_prefix} Build successful!") + logging.info("%s Build successful!", log_msg_prefix) successful_builds.append(build) continue # If it gets here, it means it is finished and not successful error_msg = f"{log_msg_prefix} Build failed!" if self.args.allow_build_failures: - logging.error(f"{error_msg}. Continuing") + logging.error("%s. Continuing", error_msg) continue - else: - logging.error(f"{error_msg}. Keeping the Launchpad repo alive") - atexit.unregister(self.delete_git_repository) - raise LaunchpadBuildFailure() + + logging.error("%s. Keeping the Launchpad repo alive", error_msg) + atexit.unregister(self.delete_git_repository) + raise LaunchpadBuildFailure() # If we got here, it means the build is still in progress # We'll keep going until len(finished_builds) >= len(build_status) @@ -401,36 +452,19 @@ def wait_for_lp_builds(self) -> list: break logging.info( - f"{len(finished_builds)}/{len(build_status)} builds finished, waiting" + "%s builds finished, waiting", + f"{len(finished_builds)}/{len(build_status)}", ) time.sleep(30) return successful_builds - def download_build_artefacts(successful_builds: list) -> None: - for build in successful_builds: - artefact_urls = build.getArtifactURLs() - rock_url = list(filter(lambda u: ".rock" in u, artefact_urls)) - if not rock_url: - arch = build.distro_arch_series_link.split("/")[-1] - logging.warning( - f"No rock artefacts found for {arch} (job {build.title})" - ) - continue - for url in rock_url: - r = requests.get(url) - r.raise_for_status() - - out_file = url.split("/")[-1] - with open(out_file, "wb") as oci_archive: - oci_archive.write(r.content) - - logging.info(f"Downloaded {out_file} into current directory") - def run(self) -> None: """Main function""" self.ack_project_will_be_public() - logging.info(f"[launchpad] Logged in as {self.lp_user} ({self.launchpad.me})") + logging.info( + "[launchpad] Logged in as %s (%s)", self.lp_user, self.launchpad.me + ) self.prepare_local_project() logging.info("Creating .launchpad.yaml file...") @@ -443,18 +477,20 @@ def run(self) -> None: f"~{self.lp_user}/+git/{self.lp_repo_name}/" ) logging.info( - f"The remote for {self.lp_repo_name} is {lp_repo_url.replace(token, '***')}" + "The remote for %s is %s", + self.lp_repo_name, + lp_repo_url.replace(token, "***"), ) try: self.push_to_lp(lp_repo_url) - except Exception: + except Exception: # pylint: disable=W0703 # Catch anything, for a graceful termination, to allow for the cleanup logging.exception("Failed to push local project to Launchpad") return logging.info( - " !! You can follow your builds at " - f"{self.lp_repo.web_link}/+ref/{self.lp_local_repo.active_branch.name} !!" + " !! You can follow your builds at %s !!", + f"{self.lp_repo.web_link}/+ref/{self.lp_local_repo.active_branch.name}", ) successful_builds = self.wait_for_lp_builds() From 878219e84a80b0d5b311759e6f78075256319c93 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Wed, 7 Feb 2024 18:43:59 +0100 Subject: [PATCH 04/14] feat: retry rock download if it fails --- rockcraft_lpci_build/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rockcraft_lpci_build/requirements.txt b/rockcraft_lpci_build/requirements.txt index c5b5ebb..bc0c098 100644 --- a/rockcraft_lpci_build/requirements.txt +++ b/rockcraft_lpci_build/requirements.txt @@ -1,4 +1,5 @@ distro-info GitPython launchpadlib -pyyaml \ No newline at end of file +pyyaml +retry \ No newline at end of file From f66abe5f81fa7a204a35a029a7712c6f44aa73f2 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Wed, 7 Feb 2024 18:44:23 +0100 Subject: [PATCH 05/14] feat: use ci_build buildstate for build monitoring --- rockcraft_lpci_build/rockcraft_lpci_build.py | 68 ++++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/rockcraft_lpci_build/rockcraft_lpci_build.py b/rockcraft_lpci_build/rockcraft_lpci_build.py index e98a202..279ffdb 100755 --- a/rockcraft_lpci_build/rockcraft_lpci_build.py +++ b/rockcraft_lpci_build/rockcraft_lpci_build.py @@ -14,7 +14,7 @@ import time from datetime import datetime, timedelta, timezone from pathlib import Path - +from typing import cast import distro_info import requests import yaml @@ -23,6 +23,7 @@ # Launchpad API docs: https://launchpad.net/+apidoc/devel.html from launchpadlib.launchpad import Launchpad from lazr.restfulclient.resource import Entry +from retry import retry # lpci reference: https://lpci.readthedocs.io/en/latest/configuration.html LPCI_CONFIG_TEMPLATE = """ @@ -60,6 +61,10 @@ class LaunchpadBuildFailure(Exception): """Custom exception for LP build failures""" +class LaunchpadBuildMissingRockArtefacts(Exception): + """Custom exception for LP builds that miss their artefacts""" + + class RockcraftLpciBuilds: """The LPCI build class""" @@ -159,38 +164,40 @@ def delete_git_repository(lp_client: Launchpad, lp_repo_path: str) -> None: git_repo.lp_delete() @staticmethod - def save_build_logs(lp_build: Entry) -> dict: + def save_build_logs(ci_build: Entry) -> None: """Fetch build logs from Launchpad and save them locally""" - ci_build = requests.get(lp_build.ci_build_link) - ci_build.raise_for_status() - ci_build = ci_build.json() - - if "build_log_url" in ci_build and ci_build["build_log_url"]: - ci_build_logs = requests.get(ci_build["build_log_url"]) + if ci_build.build_log_url: + ci_build_logs = requests.get(ci_build.build_log_url) with tempfile.NamedTemporaryFile(delete=False) as log: logging.info("Build log save at %s", log.name) log.write(ci_build_logs.text.encode()) else: logging.warning( - "Unable to get logs. build_log_url not in %s.", lp_build.ci_build_link + "Unable to get logs. build_log_url not in %s.", ci_build.web_link ) - return ci_build - @staticmethod - def download_build_artefacts(successful_builds: list) -> None: + @retry(LaunchpadBuildMissingRockArtefacts, tries=3, delay=30, backoff=2) + def get_artefact_urls(build: Entry) -> list: + """List the build artefacts, retrying if they are not immediately available""" + arch = build.distro_arch_series_link.split("/")[-1] + + artefact_urls = build.getArtifactURLs() + rock_urls = list(filter(lambda u: ".rock" in u, artefact_urls)) + logging.info("List of artefacts for %s: %s", arch, artefact_urls) + if not rock_urls: + raise LaunchpadBuildMissingRockArtefacts( + f"No rock artefacts found for {arch} (job {build.title})" + ) + + return rock_urls + + def download_build_artefacts(self, successful_builds: list) -> None: """Download rocks from the successful LP builds""" for build in successful_builds: - artefact_urls = build.getArtifactURLs() - rock_url = list(filter(lambda u: ".rock" in u, artefact_urls)) - if not rock_url: - arch = build.distro_arch_series_link.split("/")[-1] - logging.warning( - "No rock artefacts found for %s (job %s)", arch, build.title - ) - continue - for url in rock_url: + rock_urls = self.get_artefact_urls(build) + for url in rock_urls: download = requests.get(url) download.raise_for_status() @@ -425,12 +432,17 @@ def wait_for_lp_builds(self) -> list: logging.debug("%s has finished already", build.ci_build_link) continue - logging.debug("Tracking build at %s", build.ci_build_link) - if build.result in ["Failed", "Skipped", "Cancelled", "Succeeded"]: + ci_build = self.launchpad.load(build.ci_build_link) + log_msg_prefix = f"[{ci_build.arch_tag}]" + + # See buildstates at https://launchpad.net/+apidoc/devel.html#ci_build + if any( + sub_state in ci_build.buildstate.lower() + for sub_state in ["failed", "problem", "cancelled", "successfully"] + ): finished_builds.append(build.ci_build_link) - ci_build = self.save_build_logs(build) - log_msg_prefix = f"[{ci_build.get('arch_tag', 'unknown arch')}]" - if build.result == "Succeeded": + self.save_build_logs(ci_build) + if "successfully" in ci_build.buildstate.lower(): logging.info("%s Build successful!", log_msg_prefix) successful_builds.append(build) continue @@ -438,12 +450,14 @@ def wait_for_lp_builds(self) -> list: # If it gets here, it means it is finished and not successful error_msg = f"{log_msg_prefix} Build failed!" if self.args.allow_build_failures: - logging.error("%s. Continuing", error_msg) + logging.error("%s Continuing", error_msg) continue logging.error("%s. Keeping the Launchpad repo alive", error_msg) atexit.unregister(self.delete_git_repository) raise LaunchpadBuildFailure() + else: + logging.info("%s State: %s", log_msg_prefix, ci_build.buildstate) # If we got here, it means the build is still in progress # We'll keep going until len(finished_builds) >= len(build_status) From 55639599475251c802950ba34bfefb8d2443f2fa Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 14 Mar 2024 16:29:00 +0100 Subject: [PATCH 06/14] tests: add unit tests for rockcraft_lpci_build --- .github/workflows/test.yml | 13 + rockcraft_lpci_build/tests/__init__.py | 0 .../tests/test_rockcraft_lpci_build.py | 362 ++++++++++++++++++ 3 files changed, 375 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 rockcraft_lpci_build/tests/__init__.py create mode 100644 rockcraft_lpci_build/tests/test_rockcraft_lpci_build.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..efa0a4f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,13 @@ +name: Tag and Release + +on: + push: + +jobs: + test-rockcraft-lpci-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - run: pip install pytest + - run: pytest rockcraft_lpci_build/tests/ -vvv -s -rP --log-cli-level=INF \ No newline at end of file diff --git a/rockcraft_lpci_build/tests/__init__.py b/rockcraft_lpci_build/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rockcraft_lpci_build/tests/test_rockcraft_lpci_build.py b/rockcraft_lpci_build/tests/test_rockcraft_lpci_build.py new file mode 100644 index 0000000..c838641 --- /dev/null +++ b/rockcraft_lpci_build/tests/test_rockcraft_lpci_build.py @@ -0,0 +1,362 @@ +import argparse +import pathlib +import re +import sys +from unittest.mock import DEFAULT, MagicMock, call, mock_open, patch +import os +import pytest +import retry + +from rockcraft_lpci_build import rockcraft_lpci_build + + +@pytest.fixture() +def mock_cli_args(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.cli_args" + ) + + +@pytest.fixture() +def mock_set_lp_creds(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.set_lp_creds" + ) + + +@pytest.fixture() +def mock_read_rockcraft_yaml(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.read_rockcraft_yaml" + ) + + +@pytest.fixture() +def mock_lp_login(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.lp_login" + ) + + +@pytest.fixture() +def mock_os_remove(mocker): + return mocker.patch("os.remove") + + +@pytest.fixture() +def mock_lp_client(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.Launchpad") + + +@pytest.fixture() +def mock_ci_build(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.Entry") + + +@pytest.fixture() +def mock_requests(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.requests") + + +@pytest.fixture() +def mock_tempfile(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.tempfile") + + +@pytest.fixture() +def mock_get_artefact_urls(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.get_artefact_urls" + ) + + +@pytest.fixture() +def mock_builder( + mock_cli_args, mock_set_lp_creds, mock_read_rockcraft_yaml, mock_lp_login +): + return rockcraft_lpci_build.RockcraftLpciBuilds() + + +@pytest.fixture() +def mock_generic_builder(mocker): + return MagicMock() + + +@pytest.fixture() +def mock_check_rockcraft_yaml(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.check_rockcraft_yaml" + ) + + +@pytest.fixture() +def mock_atexit(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.atexit") + + +@pytest.fixture() +def mock_repo(mocker): + return mocker.patch("rockcraft_lpci_build.rockcraft_lpci_build.Repo") + + +@pytest.fixture() +def mock_get_rock_archs(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.get_rock_archs" + ) + + +@pytest.fixture() +def mock_get_rock_build_base(mocker): + return mocker.patch( + "rockcraft_lpci_build.rockcraft_lpci_build.RockcraftLpciBuilds.get_rock_build_base" + ) + + +class TestRockcraftLpciBuilds: + def test_global_attributes(self): + assert rockcraft_lpci_build.LPCI_CONFIG_TEMPLATE + + def test_attributes( + self, mock_cli_args, mock_set_lp_creds, mock_read_rockcraft_yaml, mock_lp_login + ): + obj = rockcraft_lpci_build.RockcraftLpciBuilds() + mock_cli_args.assert_called_once() + mock_set_lp_creds.assert_called_once() + assert obj.app_name == "rockcraft-lpci" + assert obj.rockcraft_yaml == pathlib.Path("rockcraft.yaml") + mock_read_rockcraft_yaml.assert_called_once() + assert obj.rock_name + mock_lp_login.assert_called_once_with("production") + assert obj.lp_user + assert obj.lp_owner + assert obj.lp_repo_name + assert obj.lp_repo_path + assert obj.lp_repo == obj.lp_local_repo == obj.lp_local_repo_path == None + assert obj.target_build_count == 0 + + def test_cli_args_missing_args(self): + with pytest.raises(SystemExit): + obj = rockcraft_lpci_build.RockcraftLpciBuilds() + obj.cli_args().parse_args() + + def test_delete_file(self, mock_os_remove): + rockcraft_lpci_build.RockcraftLpciBuilds.delete_file("foo") + mock_os_remove.assert_called_once_with("foo") + + def test_delete_git_repository(self, mock_lp_client): + mock_lp_client.git_repositories.getByPath.return_value = MagicMock() + rockcraft_lpci_build.RockcraftLpciBuilds.delete_git_repository( + mock_lp_client, "foo" + ) + mock_lp_client.git_repositories.getByPath.assert_called_once_with(path="foo") + + def test_save_build_logs(self, mock_ci_build, mock_requests, mock_tempfile): + mock_ci_build.build_log_url = None + rockcraft_lpci_build.RockcraftLpciBuilds.save_build_logs(mock_ci_build) + mock_requests.assert_not_called() + + mock_ci_build.build_log_url = "foo" + rockcraft_lpci_build.RockcraftLpciBuilds.save_build_logs(mock_ci_build) + mock_requests.get.assert_called_once_with("foo") + mock_tempfile.NamedTemporaryFile.assert_called_once_with(delete=False) + + def test_get_artefact_urls(self, mock_ci_build): + mock_ci_build.distro_arch_series_link = "foo/bar" + + def dontretry(f, *args, **kw): + return f() + + with pytest.raises(rockcraft_lpci_build.LaunchpadBuildMissingRockArtefacts): + with patch.object(retry.api, "__retry_internal", dontretry): + rockcraft_lpci_build.RockcraftLpciBuilds.get_artefact_urls( + mock_ci_build + ) + mock_ci_build.getArtifactURLs.assert_called_once() + + mock_ci_build.getArtifactURLs.return_value = ["artifact.rock", "other"] + with patch.object(retry.api, "__retry_internal", dontretry): + out = rockcraft_lpci_build.RockcraftLpciBuilds.get_artefact_urls( + mock_ci_build + ) + assert out == ["artifact.rock"] + + def test_download_build_artefacts( + self, mock_builder, mock_requests, mock_get_artefact_urls + ): + mock_builder.download_build_artefacts(successful_builds=[]) + mock_requests.assert_not_called() + mock_get_artefact_urls.assert_not_called() + + mock_builder.download_build_artefacts(successful_builds=["foo"]) + mock_get_artefact_urls.assert_called_once_with("foo") + + mock_get_artefact_urls.return_value = ["url"] + with patch("builtins.open", mock_open()) as m: + mock_builder.download_build_artefacts(successful_builds=["foo"]) + mock_requests.get.assert_called_once_with("url") + mock_requests.raise_for_status.aassert_called_once() + m.assert_called_once_with("url", "wb") + + def test_ack_project_will_be_public(self, mock_builder): + mock_builder.ack_project_will_be_public() + + with patch("builtins.input", lambda *args: "y"): + mock_builder.args.launchpad_accept_public_upload = None + mock_builder.ack_project_will_be_public() + + with patch("builtins.input", lambda *args: "n"): + with patch("sys.exit") as sysexit: + mock_builder.ack_project_will_be_public() + sysexit.assert_called_once_with(0) + + def test_read_rockcraft_yaml(self): + mock_obj = MagicMock() + mock_obj.rockcraft_yaml = "foo" + with patch("builtins.open", mock_open()) as m: + with patch("yaml.safe_load") as yaml: + rockcraft_lpci_build.RockcraftLpciBuilds.read_rockcraft_yaml(mock_obj) + mock_obj.check_rockcraft_yaml.assert_called_once() + m.assert_called_once_with("foo", "r", encoding="utf-8") + yaml.assert_called_once() + + def test_set_lp_creds(self, mock_atexit): + mock_obj = MagicMock() + mock_obj.args.lp_credentials_file = 1 + rockcraft_lpci_build.RockcraftLpciBuilds.set_lp_creds(mock_obj) + + mock_obj.args.lp_credentials_file = 0 + mock_obj.args.lp_credentials_b64 = "foo" + with patch("base64.b64decode") as base64: + with patch("tempfile.mkstemp") as tempfile: + tempfile.return_value = ("foo", "bar") + with patch("os.fdopen", mock_open()) as m: + rockcraft_lpci_build.RockcraftLpciBuilds.set_lp_creds(mock_obj) + tempfile.assert_called_once() + mock_atexit.register.assert_called_once() + base64.assert_called_once_with("foo") + + def test_lp_login(self, mock_generic_builder): + with patch("launchpadlib.launchpad.Launchpad.login_with") as login: + mock_generic_builder.rock_name = "rock" + mock_generic_builder.lp_creds = "creds" + rockcraft_lpci_build.RockcraftLpciBuilds.lp_login( + mock_generic_builder, "server" + ) + login.assert_called_once_with( + "rock remote-build", + "server", + credentials_file="creds", + credential_save_failed=mock_generic_builder.lp_login_failure, + version="devel", + ) + + def test_check_rockcraft_yaml(self, mock_builder): + with pytest.raises(FileNotFoundError): + mock_builder.check_rockcraft_yaml() + + def test_create_git_repository(self, mock_builder): + mock_builder.create_git_repository() + mock_builder.launchpad.git_repositories.new.assert_called_once() + + @patch("tempfile.mkdtemp") + @patch("os.getcwd") + @patch("os.remove") + @patch("shutil.copytree") + @patch("shutil.rmtree") + def test_prepare_local_project( + self, + mock_rmtree, + mock_copytree, + mock_remove, + mock_getcwd, + mock_mkdtemp, + mock_builder, + mock_repo, + ): + mock_builder.lp_creds = "creds" + mock_builder.prepare_local_project() + mock_mkdtemp.assert_called_once() + mock_getcwd.assert_called_once() + mock_copytree.assert_called_once() + mock_rmtree.assert_not_called() + mock_remove.assert_not_called() + mock_repo.init.assert_called_once() + + def test_get_rock_archs(self, mock_builder): + mock_builder.rockcraft_yaml_raw = {} + with pytest.raises(KeyError): + mock_builder.get_rock_archs() + + mock_builder.rockcraft_yaml_raw = {"platforms": {"amd64": ""}} + archs = mock_builder.get_rock_archs() + assert archs == ["amd64"] + + @patch("distro_info.UbuntuDistroInfo.devel") + @patch("distro_info.UbuntuDistroInfo.get_all") + def test_get_rock_build_base( + self, mock_distro_info_get_all, mock_distro_info_devel, mock_builder + ): + mock_builder.rockcraft_yaml_raw = {} + with pytest.raises(KeyError): + mock_builder.get_rock_build_base() + + mock_builder.rockcraft_yaml_raw = {"build_base": "devel"} + mock_distro_info_devel.return_value = "dev" + base = mock_builder.get_rock_build_base() + assert base == "dev" + + mock_builder.rockcraft_yaml_raw = {"base": "ubuntu@22.04"} + mock_distro_info_get_all.return_value = ["22.04", "24.04"] + base = mock_builder.get_rock_build_base() + assert base == "22.04" + mock_distro_info_devel.assert_called_once() + assert mock_distro_info_get_all.call_count == 2 + + def test_write_lpci_configuration_file( + self, mock_builder, mock_get_rock_archs, mock_get_rock_build_base + ): + with patch("yaml.safe_load") as yaml: + with patch("builtins.open", mock_open()) as m: + with patch("yaml.dump") as dump: + mock_builder.write_lpci_configuration_file() + yaml.assert_called_once_with( + rockcraft_lpci_build.LPCI_CONFIG_TEMPLATE + ) + mock_get_rock_archs.assert_called_once() + mock_get_rock_build_base.assert_called_once() + m.assert_called_once_with( + f"{mock_builder.lp_local_repo_path}/.launchpad.yaml", + "w", + encoding="utf-8", + ) + dump.assert_called_once() + + def test_get_lp_token(self, mock_builder): + mock_builder.args.timeout = 0 + mock_builder.lp_repo = MagicMock() + mock_builder.get_lp_token() + mock_builder.lp_repo.issueAccessToken.assert_called_once() + + def test_push_to_lp(self, mock_builder): + mock_builder.lp_local_repo = MagicMock() + mock_builder.lp_repo = MagicMock() + origin = MagicMock() + mock_builder.lp_local_repo.create_remote.return_value = origin + mock_builder.push_to_lp("url") + mock_builder.lp_local_repo.git.add.assert_called_once_with(A=True) + mock_builder.lp_local_repo.index.commit.assert_called_once() + mock_builder.lp_local_repo.git.checkout.assert_called_once_with("master") + mock_builder.lp_local_repo.create_remote.assert_called_once_with( + "origin", url="url" + ) + origin.push.assert_called_once() + + def test_wait_for_lp_builds(self, mock_builder, mock_atexit): + # TODO: missing tests for multiple scenarios + mock_builder.args.timeout = 1 + mock_builder.lp_local_repo = MagicMock() + mock_builder.lp_repo = MagicMock() + mock_builder.lp_repo.getStatusReports.return_value = [] + mock_builder.wait_for_lp_builds() + mock_builder.lp_repo.getStatusReports.assert_called_once() From 5fa9af1c08f964140a38deb8610968db0d09f785 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 08:53:31 +0600 Subject: [PATCH 07/14] chore(deps): update ncipollo/release-action action to v1.14.0 (#9) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7991c5b..1340fec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: echo "ARCHIVE_FILE=${ARCHIVE_FILE}" >>$GITHUB_OUTPUT - name: Create GitHub release - uses: ncipollo/release-action@v1.13.0 + uses: ncipollo/release-action@v1.14.0 if: startsWith(${{ steps.tag_version.outputs.new_tag }}, 'v') with: name: ${{ steps.tag_version.outputs.new_tag }} From aab6fe9a2673b8dc7708ea38357a90f674ffe0d6 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Tue, 13 Feb 2024 13:06:21 +0100 Subject: [PATCH 08/14] feat: add recipe and CI for rockcraft rock (#10) This includes the rockcraft.yaml, plus automations, to build, test and upload a Rockcraft rocks everytime there is a new tag in the upstream repo. --- .github/workflows/rockcraft-rock.yml | 95 +++++++++++++ .gitignore | 1 + renovate.json | 17 +++ rockcraft_rock/rockcraft.yaml | 129 ++++++++++++++++++ .../scripts/rockcraft-entrypoint.sh | 12 ++ 5 files changed, 254 insertions(+) create mode 100644 .github/workflows/rockcraft-rock.yml create mode 100644 rockcraft_rock/rockcraft.yaml create mode 100755 rockcraft_rock/scripts/rockcraft-entrypoint.sh diff --git a/.github/workflows/rockcraft-rock.yml b/.github/workflows/rockcraft-rock.yml new file mode 100644 index 0000000..6978070 --- /dev/null +++ b/.github/workflows/rockcraft-rock.yml @@ -0,0 +1,95 @@ +name: Rockcraft rock + +on: + push: + paths: + - "rockcraft_rock/rockcraft.y*ml" + +jobs: + build: + name: Build Rockcraft rock + runs-on: ubuntu-latest + outputs: + oci-archive: ${{ steps.rockcraft.outputs.rock }} + image-tag: ${{ steps.get-version.outputs.tag }} + steps: + - uses: actions/checkout@v4 + - id: get-version + run: | + tag="$(cat rockcraft_rock/rockcraft.yaml \ + | grep "source-tag: " \ + | grep -v " v"\ + | awk -F' ' '{print $NF}')" + + echo "tag=$tag" >> $GITHUB_OUTPUT + - name: Build rock + id: rockcraft + uses: canonical/craft-actions/rockcraft-pack@main + with: + path: rockcraft_rock + verbosity: debug + - uses: actions/cache/save@v3 + with: + path: ${{ steps.rockcraft.outputs.rock }} + key: ${{ github.run_id }} + + test: + name: Test Rockcraft rock + runs-on: ubuntu-latest + needs: [build] + env: + TEST_DOCKER_IMAGE: "test:latest" + steps: + - uses: actions/cache/restore@v3 + with: + path: ${{ needs.build.outputs.oci-archive }} + key: ${{ github.run_id }} + fail-on-cache-miss: true + - name: Install Skopeo + run: | + # skopeo comes inside rockcraft + sudo snap install rockcraft --classic + - run: | + /snap/rockcraft/current/bin/skopeo copy \ + oci-archive:${{ needs.build.outputs.oci-archive }} \ + docker-daemon:${{ env.TEST_DOCKER_IMAGE }} + - name: Functional test + run: | + docker run --rm ${{ env.TEST_DOCKER_IMAGE }} \ + exec /usr/libexec/rockcraft/rockcraft help + + upload: + name: Upload Rockcraft rock + runs-on: ubuntu-latest + needs: [build, test] + if: ${{ github.event_name != 'pull_request' }} + steps: + - uses: actions/cache/restore@v3 + with: + path: ${{ needs.build.outputs.oci-archive }} + key: ${{ github.run_id }} + fail-on-cache-miss: true + - name: Upload rock + uses: actions/upload-artifact@v3 + with: + name: rock + path: ${{ needs.build.outputs.oci-archive }} + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install Skopeo + run: | + # skopeo comes inside rockcraft + sudo snap install rockcraft --classic + - name: Publish rock to GHCR + run: | + /snap/rockcraft/current/bin/skopeo copy \ + oci-archive:${{ needs.build.outputs.oci-archive }} \ + docker://ghcr.io/${{ github.repository }}/rockcraft-rock:latest + + /snap/rockcraft/current/bin/skopeo copy \ + oci-archive:${{ needs.build.outputs.oci-archive }} \ + docker://ghcr.io/${{ github.repository }}/rockcraft-rock:${{ needs.build.outputs.image-tag}} diff --git a/.gitignore b/.gitignore index 9f94e5d..9b71fe7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .vscode __pycache__ +*.rock diff --git a/renovate.json b/renovate.json index 39a2b6e..a79e53c 100644 --- a/renovate.json +++ b/renovate.json @@ -2,5 +2,22 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:base" + ], + "prHourlyLimit": 0, + "ignorePaths": [ + ".github/*" + ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "^rockcraft_rock/rockcraft.yaml$" + ], + "matchStrings": [ + "source-tag: (?.*?)\\s" + ], + "datasourceTemplate": "github-releases", + "depNameTemplate": "canonical/chisel" + } ] } diff --git a/rockcraft_rock/rockcraft.yaml b/rockcraft_rock/rockcraft.yaml new file mode 100644 index 0000000..f672fb8 --- /dev/null +++ b/rockcraft_rock/rockcraft.yaml @@ -0,0 +1,129 @@ +# Run the container: +# docker run --rm -v $PWD:/project \ +# --device /dev/fuse \ +# --cap-add SYS_ADMIN \ +# --security-opt apparmor:unconfined \ +# +name: rockcraft + +# Other bases are automatically built by the CI +base: ubuntu@22.04 + +# Until adopt-info is supported, we'll just build dev images based on whatever +# is committed to the main branch +version: "dev" +summary: A Rockcraft rock +description: | + This is a rock that offers Rockcraft's capabilities from inside a container. + The default behavior is to pack a rock in destructive mode. +license: GPL-3.0 +platforms: + amd64: + +services: + rockcraft: + override: replace + startup: enabled + command: /usr/libexec/rockcraft/rockcraft-entrypoint.sh [ -v ] + working-dir: /workdir + on-success: shutdown + on-failure: shutdown + +parts: + rockcraft: + plugin: python + source: https://github.com/canonical/rockcraft.git + source-tag: 1.2.0 + python-packages: + - wheel + - pip + - setuptools + python-requirements: + - requirements-jammy.txt + - requirements.txt + build-environment: + - "CFLAGS": "$(pkg-config python-3.10 yaml-0.1 --cflags)" + build-attributes: + - enable-patchelf + build-packages: + - libapt-pkg-dev + - aspell + - aspell-en + stage-packages: + - binutils + - snapd + - python3-venv + - fuse-overlayfs + - rsync + - g++ + organize: + bin/craftctl: usr/libexec/rockcraft/craftctl + bin/rockcraft: usr/libexec/rockcraft/rockcraft + + # The custom script makes sure the build happens in a different path from + # the host's bind mount, to avoid polluting that space. + startup-script: + plugin: dump + source: scripts + organize: + rockcraft-entrypoint.sh: usr/libexec/rockcraft/rockcraft-entrypoint.sh + prime: + - usr/libexec/rockcraft/rockcraft-entrypoint.sh + + workdirs: + plugin: nil + override-build: | + # This is where Rockcraft projects on the host should be mounted + mkdir -p ${CRAFT_PART_INSTALL}/project + # This is where Rockcraft actually builds the rocks, to avoid polluting + # the host + mkdir -p ${CRAFT_PART_INSTALL}/workdir + + umoci: + plugin: make + source: https://github.com/opencontainers/umoci.git + source-tag: v0.4.7 + make-parameters: + - umoci.static + override-build: | + make umoci.static + mkdir "$CRAFT_PART_INSTALL"/bin + install -m755 umoci.static "$CRAFT_PART_INSTALL"/bin/umoci + build-packages: + - golang-go + - make + + skopeo: + plugin: nil + source: https://github.com/containers/skopeo.git + source-tag: v1.9.0 + override-build: | + CGO=1 go build -ldflags -linkmode=external ./cmd/skopeo + mkdir -p "$CRAFT_PART_INSTALL"/bin + install -m755 skopeo "$CRAFT_PART_INSTALL"/bin/skopeo + stage-packages: + - libgpgme11 + - libassuan0 + - libbtrfs0 + - libdevmapper1.02.1 + build-attributes: + - enable-patchelf + build-snaps: + - go/1.17/stable + build-packages: + - libgpgme-dev + - libassuan-dev + - libbtrfs-dev + - libdevmapper-dev + - pkg-config + overlay-packages: + - ca-certificates + + chisel: + plugin: nil + stage-snaps: + - chisel/latest/candidate + organize: + bin/chisel: usr/libexec/rockcraft/chisel + stage: + - usr/libexec/rockcraft/chisel diff --git a/rockcraft_rock/scripts/rockcraft-entrypoint.sh b/rockcraft_rock/scripts/rockcraft-entrypoint.sh new file mode 100755 index 0000000..82dc94d --- /dev/null +++ b/rockcraft_rock/scripts/rockcraft-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash -ex + +apt update &>/dev/null + +export PATH="$PATH:/usr/libexec/rockcraft" + +rsync -a --exclude="*.rock" /project/ /workdir + +/usr/libexec/rockcraft/rockcraft pack --destructive-mode "$@" + +(ls /workdir/*.rock &>/dev/null && cp /workdir/*.rock /project/) || \ + echo "No rocks were built. Exiting..." From 966adb2827e8119c74087458ac785da4d438309e Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Wed, 14 Feb 2024 14:07:25 +0100 Subject: [PATCH 09/14] fix: renovate regex for rockcraft tag (#15) --- renovate.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renovate.json b/renovate.json index a79e53c..277a3d1 100644 --- a/renovate.json +++ b/renovate.json @@ -14,10 +14,10 @@ "^rockcraft_rock/rockcraft.yaml$" ], "matchStrings": [ - "source-tag: (?.*?)\\s" + "rockcraft\\.git[\\s]*?$[\\s]*?source-tag:\\s+(?.*?)\\s" ], "datasourceTemplate": "github-releases", - "depNameTemplate": "canonical/chisel" + "depNameTemplate": "canonical/rockcraft" } ] } From 4becee75d6d52853ca66d84e037dede8d2c6b8af Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 13:43:11 +0600 Subject: [PATCH 10/14] chore(deps): update mathieudutour/github-tag-action action to v6.2 (#17) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1340fec..03159c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: - name: Bump version and push tag id: tag_version - uses: mathieudutour/github-tag-action@v6.1 + uses: mathieudutour/github-tag-action@v6.2 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.tag == '' && 'patch' || false }} From f4b665ea885a50120c95f456a431718eb9e0bce6 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 14 Mar 2024 16:29:55 +0100 Subject: [PATCH 11/14] tests: add unit tests for rockcraft_lpci_build --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efa0a4f..dde0164 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Tag and Release +name: Tests on: push: @@ -10,4 +10,4 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - run: pip install pytest - - run: pytest rockcraft_lpci_build/tests/ -vvv -s -rP --log-cli-level=INF \ No newline at end of file + - run: pytest rockcraft_lpci_build/tests/ -vvv -s -rP --log-cli-level=INFO \ No newline at end of file From 549a43b25bf05ac29f098587fe67a19892312680 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 14 Mar 2024 16:36:27 +0100 Subject: [PATCH 12/14] fix missing test dependencies --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dde0164..686b8db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,4 +10,5 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - run: pip install pytest + - run: pip install -r rockcraft_lpci_build/requirements.txt - run: pytest rockcraft_lpci_build/tests/ -vvv -s -rP --log-cli-level=INFO \ No newline at end of file From 7c269ef76abd84fc22ed635e3be89c5906118563 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 14 Mar 2024 16:38:54 +0100 Subject: [PATCH 13/14] create test dependencies file --- .github/workflows/test.yml | 3 +-- rockcraft_lpci_build/requirements.test.txt | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 rockcraft_lpci_build/requirements.test.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 686b8db..d0be3df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,5 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 - - run: pip install pytest - - run: pip install -r rockcraft_lpci_build/requirements.txt + - run: pip install -r rockcraft_lpci_build/requirements.test.txt - run: pytest rockcraft_lpci_build/tests/ -vvv -s -rP --log-cli-level=INFO \ No newline at end of file diff --git a/rockcraft_lpci_build/requirements.test.txt b/rockcraft_lpci_build/requirements.test.txt new file mode 100644 index 0000000..f9da721 --- /dev/null +++ b/rockcraft_lpci_build/requirements.test.txt @@ -0,0 +1,3 @@ +pytest +pytest-mock +retry \ No newline at end of file From 5da62bd686302b9c0b87a28c48aeb5d909aed3ab Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Thu, 14 Mar 2024 16:39:37 +0100 Subject: [PATCH 14/14] create test dependencies file --- rockcraft_lpci_build/requirements.test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rockcraft_lpci_build/requirements.test.txt b/rockcraft_lpci_build/requirements.test.txt index f9da721..888b6ed 100644 --- a/rockcraft_lpci_build/requirements.test.txt +++ b/rockcraft_lpci_build/requirements.test.txt @@ -1,3 +1,4 @@ pytest pytest-mock -retry \ No newline at end of file +retry +GitPython \ No newline at end of file