fix(CLDX-190): changes as per testing sign-binaries task with pipeline

pipelines/push-binaries-to-dev-portal/README.md

@@ -19,6 +19,9 @@ Tekton pipeline to sign and release Red Hat binaries to the Red Hat Developer Po
 | taskGitUrl                      | The url to the git repo where the release-service-catalog tasks to be used are stored                  | Yes      | https://github.com/konflux-ci/release-service-catalog.git |
 | taskGitRevision                 | The revision in the taskGitUrl repo to be used                                                         | No       | -             |
 
+## Changes in 1.4.0
+* Use results from `sign-binaries` task for `push-to-cdn` and `publish-to-cgw` instead of `extract-binaries-from-image`
+
 ## Changes in 1.3.0
 * Add more params for `sign-binaries` task
 

pipelines/push-binaries-to-dev-portal/push-binaries-to-dev-portal.yaml

@@ -4,7 +4,7 @@ kind: Pipeline
 metadata:
   name: push-binaries-to-dev-portal
   labels:
-    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/version: "1.4.0"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -264,7 +264,7 @@ spec:
         - name: exodusGwSecret
           value: $(tasks.prepare-exodus-params.results.exodusGwSecret)
         - name: binariesDir
-          value: $(tasks.extract-binaries-from-image.results.binaries_path)
+          value: $(tasks.sign-binaries.results.binaries_path)
         - name: subdirectory
           value: $(context.pipelineRun.uid)
       runAfter:
@@ -286,7 +286,7 @@ spec:
         - name: dataPath
           value: "$(tasks.collect-data.results.data)"
         - name: contentDir
-          value: $(tasks.extract-binaries-from-image.results.binaries_path)
+          value: $(tasks.sign-binaries.results.binaries_path)
       runAfter:
         - push-to-cdn
   finally:

tasks/extract-binaries-from-image/README.md

@@ -17,6 +17,9 @@ passed.
 | subdirectory        | Subdirectory inside the workspace to be used for storing the binaries      | Yes      | ""            |
 | dataPath            | Path to the JSON string of the merged data to use in the data workspace    | No       | -             |
 
+## Changes in 2.1.2
+* Fix typo in fetching DESIRED_COMPONENTS_LIST `content-gateway` -> `contentGateway`
+
 ## Changes in 2.1.1
 * Fix shellcheck/checkton linting issues in the task and tests
 

tasks/extract-binaries-from-image/extract-binaries-from-image.yaml

@@ -4,7 +4,7 @@ kind: Task
 metadata:
   name: extract-binaries-from-image
   labels:
-    app.kubernetes.io/version: "2.1.1"
+    app.kubernetes.io/version: "2.1.2"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -55,8 +55,8 @@ spec:
         DESIRED_COMPONENTS_LIST=
         if [ ! -f "${DATA_FILE}" ] ; then
             echo "No data JSON was provided."
-        elif [ "$(jq '."content-gateway" | has("components")' "${DATA_FILE}")" = true ]; then
-            DESIRED_COMPONENTS_LIST="$(jq -r '."content-gateway".components[].name' "${DATA_FILE}")"
+        elif [ "$(jq '."contentGateway" | has("components")' "${DATA_FILE}")" = true ]; then
+            DESIRED_COMPONENTS_LIST="$(jq -r '."contentGateway".components[].name' "${DATA_FILE}")"
         fi
 
         NUM_COMPONENTS=$(jq '.components | length' "$SNAPSHOT_SPEC_FILE")

tasks/extract-binaries-from-image/tests/test-extract-binaries-multiple-components-desired-only.yaml

@@ -46,10 +46,10 @@ spec:
 
               cat > "$(workspaces.data.path)"/data.json << EOF
               {
-                "content-gateway": {
+                "contentGateway": {
                   "components": [
-                    {"name": "comp"},
-                    {"name": "comp2"}
+                    {"name": "comp2"},
+                    {"name": "comp3"}
                   ]
                 }
               }

tasks/prepare-exodus-params/README.md

@@ -1,14 +1,19 @@
 # prepare-exodus-params
 
 Tekton task that prepares exodus configuration options from the data file. The task looks at the data file
-in the workspace to extract the `exodus.env` key. Depending on the values, correct configuration options are returned. 
+in the workspace to extract the `cdn.env` key. Depending on the values, correct configuration options are returned. 
+
+exodus-gw is a microservice dedicated to writing data onto the CDN.
 
 ## Parameters
 
 | Name     | Description                                                                                                                 | Optional | Default value |
 |----------|-----------------------------------------------------------------------------------------------------------------------------|----------|---------------|
 | dataPath | Path to the merged data JSON file generated by collect-data task and containing the exodus configuration options to use     | No       | -             |
 
+## Changes in 0.2.2
+* Use `cdn.env` instead of `exodus.env` to align RPA data with other pipelines
+
 ## Changes in 0.2.1
 * remove unnecessary new line before outputting the result
 

tasks/prepare-exodus-params/prepare-exodus-params.yaml

@@ -4,13 +4,13 @@ kind: Task
 metadata:
   name: prepare-exodus-params
   labels:
-    app.kubernetes.io/version: "0.2.1"
+    app.kubernetes.io/version: "0.2.2"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
 spec:
   description: >-
-    Tekton task that prepares exodus configuration options from the data file
+    Tekton task that prepares exodus (CDN) configuration options from the data file
   params:
     - name: dataPath
       type: string
@@ -44,26 +44,28 @@ spec:
             exit 1
         fi
 
-        if [ $(jq '.exodus | has("env")' $DATA_FILE) == false ] ; then
-            echo "Exodus env missing in data JSON file"
+        if [ "$(jq '.cdn | has("env")' "${DATA_FILE}")" = "false" ] ; then
+            echo "CDN env missing in data JSON file"
             exit 1
         fi
 
-        EXODUS_ENV=$(jq -r '.exodus.env' $DATA_FILE)
+        EXODUS_ENV=$(jq -r '.cdn.env' "$DATA_FILE")
         EXODUS_GW_URL="https://exodus-gw.corp.redhat.com"
         EXODUS_GW_ENV="live"
         EXODUS_GW_SECRET="exodus-prod-secret"
 
-        # Check if EXODUS_ENV contains "cdn-qa"
-        if [[ "$EXODUS_ENV" == *"cdn-qa"* ]]; then
-          EXODUS_GW_SECRET="exodus-stage-secret"
-          EXODUS_GW_URL="https://exodus-gw.corp.stage.redhat.com"
-        fi
+        case "$EXODUS_ENV" in 
+          *qa*)
+            EXODUS_GW_SECRET="exodus-stage-secret"
+            EXODUS_GW_URL="https://exodus-gw.corp.stage.redhat.com"
+            ;;
+        esac
 
-        # Check if EXODUS_ENV contains "cdn-stage"
-        if [[ "$EXODUS_ENV" == *"cdn-stage"* ]]; then
-          EXODUS_GW_ENV="pre"
-        fi
+        case "$EXODUS_ENV" in
+          *stage*)
+            EXODUS_GW_ENV="pre"
+            ;;
+        esac
 
         echo -n $EXODUS_GW_SECRET > $(results.exodusGwSecret.path)
         echo -n $EXODUS_GW_URL > $(results.exodusGwUrl.path)

tasks/prepare-exodus-params/tests/test-prepare-exodus-params-cdn-live.yaml

@@ -25,8 +25,8 @@ spec:
 
               cat > $(workspaces.data.path)/data.json << EOF
               {
-                "exodus": {
-                  "env": "cdn-live"
+                "cdn": {
+                  "env": "production"
                 }
               }
               EOF

tasks/prepare-exodus-params/tests/test-prepare-exodus-params-cdn-qa.yaml

@@ -25,8 +25,8 @@ spec:
 
               cat > $(workspaces.data.path)/data.json << EOF
               {
-                "exodus": {
-                  "env": "cdn-qa"
+                "cdn": {
+                  "env": "qa"
                 }
               }
               EOF

tasks/prepare-exodus-params/tests/test-prepare-exodus-params-cdn-stage.yaml

@@ -25,8 +25,8 @@ spec:
 
               cat > $(workspaces.data.path)/data.json << EOF
               {
-                "exodus": {
-                  "env": "cdn-stage"
+                "cdn": {
+                  "env": "stage"
                 }
               }
               EOF

tasks/publish-to-cgw/README.md

@@ -16,6 +16,10 @@ Tekton task to publish content to Red Hat's Developer portal using pubtools-cont
 | cgwHostname | The hostname of the content-gateway to publish the metadata to  | yes      | https://developers.redhat.com/content-gateway/rest/admin |
 | cgwSecret   | The kubernetes secret to use to authenticate to content-gateway | yes      | publish-to-cgw-secret |
 
+## Changes in 0.2.4
+* Raise Exception correctly when `pubtools-content-gateway` fails
+* Create unique shortURL for each product version
+
 ## Changes in 0.2.3
 * Added logic to handle checksum files
 * Fix bug in computing shortUrl

tasks/publish-to-cgw/publish-to-cgw.yaml

@@ -4,7 +4,7 @@ kind: Task
 metadata:
   name: publish-to-cgw
   labels:
-    app.kubernetes.io/version: "0.2.3"
+    app.kubernetes.io/version: "0.2.4"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -114,34 +114,36 @@ spec:
             Generate metadata for each file in
             content_list that starts with the component name
             """
+            shortURL_base = '/pub/cgw'
             metadata = []
+            shasum_files_processed = []
             for file in content_list:
-                matching_component = [
-                    data
-                    for data in components
-                    if file.startswith(data['name'])]
+                matching_component = None
+                for component in components:
+                    if file.startswith(component['name']):
+                        matching_component = component.copy()
+                        break
 
                 if matching_component:
                     print("Processing file: ", file)
-                    for data in matching_component:
-                        component = data.copy()
-                        component.update({
-                            'productName': productName,
-                            'productCode': productCode,
-                            'productVersionName': productVersionName,
-                            'downloadURL': generate_download_url(file),
-                            'shortURL': f"/cgw/{productCode}/{file}",
-                            'label': file,
-                        })
-                        del component['name']
-                        metadata.append({
-                            'type': 'file',
-                            'action': 'create',
-                            'metadata': {**default_values_per_component, **component}
-                        })
-
+                    matching_component.update({
+                        'productName': productName,
+                        'productCode': productCode,
+                        'productVersionName': productVersionName,
+                        'downloadURL': generate_download_url(file),
+                        'shortURL': f"{shortURL_base}/{productCode}/{productVersionName}/{file}",
+                        'label': file,
+                    })
+                    del matching_component['name']
+                    metadata.append({
+                        'type': 'file',
+                        'action': 'create',
+                        'metadata': {**default_values_per_component, **matching_component}
+                    })
                 else:
-                    if file.startswith('sha256'):
+                    if file.startswith('sha256') and file not in shasum_files_processed:
+                        shasum_files_processed.append(file)
+                        print("Processing file: ", file)
                         if file.endswith(".gpg"):
                             label = "Checksum - GPG"
                         elif file.endswith(".sig"):
@@ -157,7 +159,7 @@ spec:
                                 'productCode': productCode,
                                 'productVersionName': productVersionName,
                                 'downloadURL': generate_download_url(file),
-                                'shortURL': f"/cgw/{productCode}/{file}",
+                                'shortURL': f"{shortURL_base}/{productCode}/{productVersionName}/{file}",
                                 'label': label,
                                 **default_values_per_component
                             }
@@ -187,12 +189,15 @@ spec:
         ]
 
         try:
-            result = subprocess.run(command, capture_output=True, text=True)
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
             command_output = result.stderr # using stderr to capture logged output
             print(f"Command succeeded with {command_output}")
         except subprocess.CalledProcessError as error:
-            print(f"Command failed with return code {error.returncode}")
+            print(f"Command failed with return code {error.returncode}\n")
+            print(f"CGW metadata that was passed: {metadata}\n")
             command_output = error.stderr
+            print(f" ERROR:\n{command_output}")
+            raise
 
         result_data = {"no_of_files_processed": len(metadata), 
                       "metadata_file_path": METADATA_FILE_PATH,