From cb048069ad8c198c230d9359cc0655f426c2fb2b Mon Sep 17 00:00:00 2001 From: pkhander Date: Wed, 23 Oct 2024 10:19:00 -0400 Subject: [PATCH] fix(CLDX-190): changes as per testing sign-binaries task with pipeline This commit fixes tasks releated to push-to-dev-portal pipeline Signed-off-by: Parthey Khanderia --- .../push-binaries-to-dev-portal/README.md | 3 + .../push-binaries-to-dev-portal.yaml | 6 +- tasks/extract-binaries-from-image/README.md | 3 + .../extract-binaries-from-image.yaml | 4 +- tasks/prepare-exodus-params/README.md | 7 ++- .../prepare-exodus-params.yaml | 14 ++--- tasks/publish-to-cgw/README.md | 4 ++ tasks/publish-to-cgw/publish-to-cgw.yaml | 57 ++++++++++--------- 8 files changed, 59 insertions(+), 39 deletions(-) diff --git a/pipelines/push-binaries-to-dev-portal/README.md b/pipelines/push-binaries-to-dev-portal/README.md index 71378762a..23d1ae69c 100644 --- a/pipelines/push-binaries-to-dev-portal/README.md +++ b/pipelines/push-binaries-to-dev-portal/README.md @@ -19,6 +19,9 @@ Tekton pipeline to sign and release Red Hat binaries to the Red Hat Developer Po | taskGitUrl | The url to the git repo where the release-service-catalog tasks to be used are stored | Yes | https://github.com/konflux-ci/release-service-catalog.git | | taskGitRevision | The revision in the taskGitUrl repo to be used | No | - | +## Changes in 1.4.0 +* Use results from `sign-binaries` task for `push-to-cdn` and `publish-to-cgw` instead of `extract-binaries-from-image` + ## Changes in 1.3.0 * Add more params for `sign-binaries` task diff --git a/pipelines/push-binaries-to-dev-portal/push-binaries-to-dev-portal.yaml b/pipelines/push-binaries-to-dev-portal/push-binaries-to-dev-portal.yaml index a35bcd23f..b177491ec 100644 --- a/pipelines/push-binaries-to-dev-portal/push-binaries-to-dev-portal.yaml +++ b/pipelines/push-binaries-to-dev-portal/push-binaries-to-dev-portal.yaml @@ -4,7 +4,7 @@ kind: Pipeline metadata: name: push-binaries-to-dev-portal labels: - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.4.0" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release @@ -264,7 +264,7 @@ spec: - name: exodusGwSecret value: $(tasks.prepare-exodus-params.results.exodusGwSecret) - name: binariesDir - value: $(tasks.extract-binaries-from-image.results.binaries_path) + value: $(tasks.sign-binaries.results.binaries_path) - name: subdirectory value: $(context.pipelineRun.uid) runAfter: @@ -286,7 +286,7 @@ spec: - name: dataPath value: "$(tasks.collect-data.results.data)" - name: contentDir - value: $(tasks.extract-binaries-from-image.results.binaries_path) + value: $(tasks.sign-binaries.results.binaries_path) runAfter: - push-to-cdn finally: diff --git a/tasks/extract-binaries-from-image/README.md b/tasks/extract-binaries-from-image/README.md index 209905ae5..d838b108e 100644 --- a/tasks/extract-binaries-from-image/README.md +++ b/tasks/extract-binaries-from-image/README.md @@ -17,6 +17,9 @@ passed. | subdirectory | Subdirectory inside the workspace to be used for storing the binaries | Yes | "" | | dataPath | Path to the JSON string of the merged data to use in the data workspace | No | - | +## Changes in 2.1.2 +* Fix typo in fetching DESIRED_COMPONENTS_LIST `content-gateway` -> `contentGateway` + ## Changes in 2.1.1 * Fix shellcheck/checkton linting issues in the task and tests diff --git a/tasks/extract-binaries-from-image/extract-binaries-from-image.yaml b/tasks/extract-binaries-from-image/extract-binaries-from-image.yaml index a16845574..7638b0180 100644 --- a/tasks/extract-binaries-from-image/extract-binaries-from-image.yaml +++ b/tasks/extract-binaries-from-image/extract-binaries-from-image.yaml @@ -55,8 +55,8 @@ spec: DESIRED_COMPONENTS_LIST= if [ ! -f "${DATA_FILE}" ] ; then echo "No data JSON was provided." - elif [ "$(jq '."content-gateway" | has("components")' "${DATA_FILE}")" = true ]; then - DESIRED_COMPONENTS_LIST="$(jq -r '."content-gateway".components[].name' "${DATA_FILE}")" + elif [ "$(jq '."contentGateway" | has("components")' "${DATA_FILE}")" = true ]; then + DESIRED_COMPONENTS_LIST="$(jq -r '."contentGateway".components[].name' "${DATA_FILE}")" fi NUM_COMPONENTS=$(jq '.components | length' "$SNAPSHOT_SPEC_FILE") diff --git a/tasks/prepare-exodus-params/README.md b/tasks/prepare-exodus-params/README.md index bbf534ac5..8e6cf6ad3 100644 --- a/tasks/prepare-exodus-params/README.md +++ b/tasks/prepare-exodus-params/README.md @@ -1,7 +1,9 @@ # prepare-exodus-params Tekton task that prepares exodus configuration options from the data file. The task looks at the data file -in the workspace to extract the `exodus.env` key. Depending on the values, correct configuration options are returned. +in the workspace to extract the `cdn.env` key. Depending on the values, correct configuration options are returned. + +exodus-gw is a microservice dedicated to writing data onto the CDN. ## Parameters @@ -9,6 +11,9 @@ in the workspace to extract the `exodus.env` key. Depending on the values, corre |----------|-----------------------------------------------------------------------------------------------------------------------------|----------|---------------| | dataPath | Path to the merged data JSON file generated by collect-data task and containing the exodus configuration options to use | No | - | +## Changes in 0.2.2 +* Use `cdn.env` instead of `exodus.env` to align RPA data with other pipelines + ## Changes in 0.2.1 * remove unnecessary new line before outputting the result diff --git a/tasks/prepare-exodus-params/prepare-exodus-params.yaml b/tasks/prepare-exodus-params/prepare-exodus-params.yaml index 4619df971..9d5192569 100644 --- a/tasks/prepare-exodus-params/prepare-exodus-params.yaml +++ b/tasks/prepare-exodus-params/prepare-exodus-params.yaml @@ -4,13 +4,13 @@ kind: Task metadata: name: prepare-exodus-params labels: - app.kubernetes.io/version: "0.2.1" + app.kubernetes.io/version: "0.2.2" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release spec: description: >- - Tekton task that prepares exodus configuration options from the data file + Tekton task that prepares exodus (CDN) configuration options from the data file params: - name: dataPath type: string @@ -44,24 +44,24 @@ spec: exit 1 fi - if [ $(jq '.exodus | has("env")' $DATA_FILE) == false ] ; then - echo "Exodus env missing in data JSON file" + if [ $(jq '.cdn | has("env")' $DATA_FILE) == false ] ; then + echo "CDN env missing in data JSON file" exit 1 fi - EXODUS_ENV=$(jq -r '.exodus.env' $DATA_FILE) + EXODUS_ENV=$(jq -r '.cdn.env' $DATA_FILE) EXODUS_GW_URL="https://exodus-gw.corp.redhat.com" EXODUS_GW_ENV="live" EXODUS_GW_SECRET="exodus-prod-secret" # Check if EXODUS_ENV contains "cdn-qa" - if [[ "$EXODUS_ENV" == *"cdn-qa"* ]]; then + if [[ "$EXODUS_ENV" == *"qa"* ]]; then EXODUS_GW_SECRET="exodus-stage-secret" EXODUS_GW_URL="https://exodus-gw.corp.stage.redhat.com" fi # Check if EXODUS_ENV contains "cdn-stage" - if [[ "$EXODUS_ENV" == *"cdn-stage"* ]]; then + if [[ "$EXODUS_ENV" == *"stage"* ]]; then EXODUS_GW_ENV="pre" fi diff --git a/tasks/publish-to-cgw/README.md b/tasks/publish-to-cgw/README.md index 20b433de5..a10ebc1d1 100644 --- a/tasks/publish-to-cgw/README.md +++ b/tasks/publish-to-cgw/README.md @@ -16,6 +16,10 @@ Tekton task to publish content to Red Hat's Developer portal using pubtools-cont | cgwHostname | The hostname of the content-gateway to publish the metadata to | yes | https://developers.redhat.com/content-gateway/rest/admin | | cgwSecret | The kubernetes secret to use to authenticate to content-gateway | yes | publish-to-cgw-secret | +## Changes in 0.2.3 +* Raise Exception correctly when `pubtools-content-gateway` fails +* Create unique shortURL for each product version + ## Changes in 0.2.3 * Added logic to handle checksum files * Fix bug in computing shortUrl diff --git a/tasks/publish-to-cgw/publish-to-cgw.yaml b/tasks/publish-to-cgw/publish-to-cgw.yaml index 9ea85c8ee..a5e7031ba 100644 --- a/tasks/publish-to-cgw/publish-to-cgw.yaml +++ b/tasks/publish-to-cgw/publish-to-cgw.yaml @@ -4,7 +4,7 @@ kind: Task metadata: name: publish-to-cgw labels: - app.kubernetes.io/version: "0.2.3" + app.kubernetes.io/version: "0.2.4" annotations: tekton.dev/pipelines.minVersion: "0.12.1" tekton.dev/tags: release @@ -114,34 +114,36 @@ spec: Generate metadata for each file in content_list that starts with the component name """ + shortURL_base = '/pub/cgw' metadata = [] + shasum_files_processed = [] for file in content_list: - matching_component = [ - data - for data in components - if file.startswith(data['name'])] + matching_component = None + for component in components: + if file.startswith(component['name']): + matching_component = component.copy() + break if matching_component: print("Processing file: ", file) - for data in matching_component: - component = data.copy() - component.update({ - 'productName': productName, - 'productCode': productCode, - 'productVersionName': productVersionName, - 'downloadURL': generate_download_url(file), - 'shortURL': f"/cgw/{productCode}/{file}", - 'label': file, - }) - del component['name'] - metadata.append({ - 'type': 'file', - 'action': 'create', - 'metadata': {**default_values_per_component, **component} - }) - + matching_component.update({ + 'productName': productName, + 'productCode': productCode, + 'productVersionName': productVersionName, + 'downloadURL': generate_download_url(file), + 'shortURL': f"{shortURL_base}/{productCode}/{productVersionName}/{file}", + 'label': file, + }) + del matching_component['name'] + metadata.append({ + 'type': 'file', + 'action': 'create', + 'metadata': {**default_values_per_component, **matching_component} + }) else: - if file.startswith('sha256'): + if file.startswith('sha256') and file not in shasum_files_processed: + shasum_files_processed.append(file) + print("Processing file: ", file) if file.endswith(".gpg"): label = "Checksum - GPG" elif file.endswith(".sig"): @@ -157,7 +159,7 @@ spec: 'productCode': productCode, 'productVersionName': productVersionName, 'downloadURL': generate_download_url(file), - 'shortURL': f"/cgw/{productCode}/{file}", + 'shortURL': f"{shortURL_base}/{productCode}/{productVersionName}/{file}", 'label': label, **default_values_per_component } @@ -187,12 +189,15 @@ spec: ] try: - result = subprocess.run(command, capture_output=True, text=True) + result = subprocess.run(command, capture_output=True, text=True, check=True) command_output = result.stderr # using stderr to capture logged output print(f"Command succeeded with {command_output}") except subprocess.CalledProcessError as error: - print(f"Command failed with return code {error.returncode}") + print(f"Command failed with return code {error.returncode}\n") + print(f"CGW metadata that was passed: {metadata}\n") command_output = error.stderr + print(f" ERROR:\n{command_output}") + raise result_data = {"no_of_files_processed": len(metadata), "metadata_file_path": METADATA_FILE_PATH,