diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 5c7636210..999c1e24c 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -514,12 +514,53 @@ functions: - rh_pyxis binary: scripts/dev/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}" + install_macnotary: + - command: shell.exec + type: setup + params: + include_expansions_in_env: + - notary_service_url + script: | + set -Eeu pipefail + + curl "${notary_service_url}" --output macos-notary.zip + unzip -u macos-notary.zip + chmod 755 ./linux_amd64/macnotary + + install_goreleaser: + - command: shell.exec + type: setup + include_expansions_in_env: + - goreleaser_pro_tar_gz + params: + working_dir: src/github.com/mongodb/mongodb-kubernetes + script: | + set -Eeu pipefail + curl -fL "${goreleaser_pro_tar_gz}" --output goreleaser_Linux_x86_64.tar.gz + tar -xf goreleaser_Linux_x86_64.tar.gz + chmod 755 ./goreleaser + build_multi_cluster_binary: - command: subprocess.exec type: setup params: + include_expansions_in_env: + - github_commit + - GRS_USERNAME + - GRS_PASSWORD + - PKCS11_URI + - ARTIFACTORY_URL + - ARTIFACTORY_PASSWORD + - SIGNING_IMAGE_URI + - macos_notary_keyid + - macos_notary_secret + - workdir + - triggered_by_git_tag + env: + MACOS_NOTARY_KEY: ${macos_notary_keyid} + MACOS_NOTARY_SECRET: ${macos_notary_secret} working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh + binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py build_and_push_appdb_database: - command: subprocess.exec diff --git a/.evergreen.yml b/.evergreen.yml index b1ab8a9bf..d0ea9bfc2 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -84,6 +84,8 @@ variables: - func: clone - func: download_kube_tools - func: setup_building_host + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary - &setup_and_teardown_group_gke_code_snippets @@ -93,6 +95,8 @@ variables: - func: setup_gcloud_cli - func: setup_mongosh - func: download_kube_tools + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary teardown_group: - func: upload_code_snippets_logs @@ -411,6 +415,8 @@ tasks: commands: - func: clone - func: setup_building_host + - func: install_goreleaser + - func: install_macnotary - func: build_multi_cluster_binary - func: pipeline vars: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 70b220d43..884ae3fc4 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -14,6 +14,8 @@ builds: goarch: - amd64 - arm64 + - s390x + - ppc64le hooks: # This will notarize Apple binaries and replace goreleaser bins with the notarized ones post: diff --git a/build_info.json b/build_info.json index 6bb5ddd50..30e02eb97 100644 --- a/build_info.json +++ b/build_info.json @@ -298,14 +298,14 @@ "binaries": { "kubectl-mongodb": { "patch": { - "s3-store": "s3://kubectl-mongodb/dev", + "s3-store": "mongodb-kubernetes-dev", "platforms": [ "linux/amd64" ] }, "staging": { - "sign": true, - "s3-store": "s3://kubectl-mongodb/staging", + "sign": false, + "s3-store": "mongodb-kubernetes-staging", "platforms": [ "darwin/amd64", "darwin/arm64", @@ -315,7 +315,7 @@ }, "release": { "sign": true, - "s3-store": "s3://kubectl-mongodb/prod", + "s3-store": "mongodb-kubernetes-release", "platforms": [ "darwin/amd64", "darwin/arm64", diff --git a/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py new file mode 100755 index 000000000..9be77a4a9 --- /dev/null +++ b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py @@ -0,0 +1,156 @@ +import os +import subprocess +import sys + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError + +from lib.base_logger import logger +from scripts.release.build.build_info import ( + load_build_info, +) +from scripts.release.build.build_scenario import ( + BuildScenario, +) + +AWS_REGION = "eu-north-1" +KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb" +S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME + +GORELEASER_DIST_DIR = "dist" + + +def run_goreleaser(): + try: + command = ["./goreleaser", "build", "--snapshot", "--clean", "--skip", "post-hooks"] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + + for log in iter(process.stdout.readline, ""): + print(log, end="") + + process.stdout.close() + exit_code = process.wait() + + if exit_code != 0: + logger.debug(f"GoReleaser command failed with exit code {exit_code}.") + sys.exit(1) + + logger.info("GoReleaser build completed successfully!") + + except FileNotFoundError: + logger.debug( + "ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH." + ) + sys.exit(1) + except Exception as e: + logger.debug(f"An unexpected error occurred while running `goreleaser build`: {e}") + sys.exit(1) + + +# upload_artifacts_to_s3 uploads the artifacts that are generated by goreleaser to S3 bucket at a specific path. +# The S3 bucket and version are figured out and passed to this function based on BuildScenario. +def upload_artifacts_to_s3(s3_bucket: str, version: str): + if not os.path.isdir(GORELEASER_DIST_DIR): + logger.info(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.") + sys.exit(1) + + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except (NoCredentialsError, PartialCredentialsError): + logger.debug("ERROR: Failed to create S3 client. AWS credentials not found.") + sys.exit(1) + except Exception as e: + logger.debug(f"An error occurred connecting to S3: {e}") + sys.exit(1) + + uploaded_files = 0 + # iterate over all the files generated by goreleaser in the dist directory and upload them to S3 + for subdir in os.listdir(GORELEASER_DIST_DIR): + subdir_path = os.path.join(GORELEASER_DIST_DIR, subdir) + if not os.path.isdir(subdir_path): + continue # not a directory + + for filename in os.listdir(subdir_path): + local_file_path = os.path.join(subdir_path, filename) + if not os.path.isfile(local_file_path): + continue + + s3_key = s3_path(local_file_path, version) + logger.info(f"Uploading artifact {local_file_path} to s3://{s3_bucket}/{s3_key}") + try: + s3_client.upload_file(local_file_path, s3_bucket, s3_key) + logger.info(f"Successfully uploaded the artifact {filename}") + uploaded_files += 1 + except Exception as e: + logger.debug(f"ERROR: Failed to upload file {filename}: {e}") + sys.exit(1) + + if uploaded_files > 0: + logger.info(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.") + + +# s3_path returns the path where the artifacts should be uploaded to in S3 object store. +# For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/{goreleaser-artifact}`, +# for staging workflows it would be `kubectl-mongodb/{commit-sha}/{goreleaser-artifact}`. +# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario. +def s3_path(local_path: str, version: str): + return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/{local_path}" + + +def s3_and_local_plugin_path(version: str) -> dict[str, str]: + s3_common_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist" + local_common_path = "docker/mongodb-kubernetes-tests" + # path in s3 : local path + return { + f"{s3_common_path}/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_amd64", + f"{s3_common_path}/kubectl-mongodb_linux_arm64/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_arm64", + f"{s3_common_path}/kubectl-mongodb_linux_ppc64le/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_ppc64le", + f"{s3_common_path}/kubectl-mongodb_linux_s390x/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_s390x", + } + + +# download_plugin_for_tests_image downloads the plugin for all the architectures and places them to the paths configured in +# s3_and_local_plugin_path +def download_plugin_for_tests_image(build_scenario: BuildScenario, s3_bucket: str, version: str): + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except Exception as e: + logger.debug(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}") + sys.exit(1) + + plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb" + for plugin_path, local_path in s3_and_local_plugin_path(version).items(): + logger.info(f"Downloading s3://{s3_bucket}/{plugin_path} to {local_path}") + try: + s3_client.download_file(s3_bucket, plugin_path, local_path) + # change the file's permissions to make file executable + os.chmod(local_path, 0o755) + + logger.info(f"Successfully downloaded artifact to {local_path}") + except ClientError as e: + if e.response["Error"]["Code"] == "404": + logger.debug(f"ERROR: Artifact not found at s3://{s3_bucket}/{plugin_path} ") + else: + logger.debug(f"ERROR: Failed to download artifact. S3 Client Error: {e}") + sys.exit(1) + except Exception as e: + logger.debug(f"An unexpected error occurred during download: {e}") + sys.exit(1) + + +def main(): + build_scenario = BuildScenario.infer_scenario_from_environment() + kubectl_plugin_build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY_NAME] + + run_goreleaser() + + upload_artifacts_to_s3(kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version) + + download_plugin_for_tests_image( + build_scenario, kubectl_plugin_build_info.s3_store, kubectl_plugin_build_info.version + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index e3c341da7..9fc85f91a 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -106,7 +106,7 @@ def test_load_build_info_development(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], version=version, sign=False, @@ -221,7 +221,7 @@ def test_load_build_info_patch(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], version=patch_id, sign=False, @@ -339,10 +339,10 @@ def test_load_build_info_staging(git_repo: Repo): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/staging", + s3_store="mongodb-kubernetes-staging", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], version=expected_commit_sha, - sign=True, + sign=False, ) }, helm_charts={ @@ -419,7 +419,7 @@ def test_load_build_info_release( }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/prod", + s3_store="mongodb-kubernetes-release", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], version=version, sign=True,