Revert "Merge branch 'ginkgo_removal' of https://github.com/test-netw…

…ork-function/cnf-certification-test into ginkgo_removal"

This reverts commit 037fb46, reversing
changes made to 254f3e0.

.github/actions/setup/action.yml

@@ -17,7 +17,6 @@ runs:
       uses: actions/setup-go@v4
       with:
         go-version: 1.21.4
-        cache: false
 
     - name: Disable default go problem matcher
       run: echo "::remove-matcher owner=go::"

.github/workflows/pre-main.yaml

@@ -451,7 +451,7 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to create \`unstable\` container image from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.GCHAT_WEBHOOK_URL }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.GCHAT_WEBHOOK_URL }}
 
       # Push the new unstable TNF image to Quay.io.
       - name: (if on main and upstream) Authenticate against Quay.io

.github/workflows/qe-hosted.yml

@@ -1,8 +1,8 @@
 name: QE Testing (Ubuntu-hosted)
 
 on:
-  pull_request:
-    branches: [ main ]
+  # pull_request:
+  #   branches: [ main ]
   workflow_dispatch:
   # Schedule a daily cron at midnight UTC
   schedule:
@@ -26,7 +26,6 @@ jobs:
       TEST_TNF_IMAGE_NAME: quay.io/testnetworkfunction/cnf-certification-test
       TEST_TNF_IMAGE_TAG: localtest
       DOCKER_CONFIG_DIR: '/home/runner/.docker/'
-      SKIP_PRELOAD_IMAGES: true
 
     steps:
       - name: Free Disk Space (Ubuntu)
@@ -41,6 +40,20 @@ jobs:
           swap-storage: true
         continue-on-error: true
 
+      - name: Write temporary docker file
+        run: |
+          mkdir -p /home/runner/.docker
+          touch ${PFLT_DOCKERCONFIG}
+          echo '{ "auths": {} }' >> ${PFLT_DOCKERCONFIG}
+
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.4
+
+      - name: Disable default go problem matcher
+        run: echo "::remove-matcher owner=go::"
+
       - name: Check out code
         uses: actions/checkout@v4
         with:
@@ -90,12 +103,10 @@ jobs:
           repository: ${{ env.QE_REPO }}
           path: cnfcert-tests-verification
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -107,4 +118,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run Github hosted QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/qe-ocp-413-intrusive.yaml

@@ -62,12 +62,10 @@ jobs:
           sudo rm -rf /tmp/tnf_config/
           sudo rm -rf /tmp/tnf_report/
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=false ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=false ENABLE_PARALLEL=false ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -79,4 +77,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run intrusive OCP 4.13 QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/qe-ocp-413.yaml

@@ -61,12 +61,10 @@ jobs:
           sudo rm -rf /tmp/tnf_config/
           sudo rm -rf /tmp/tnf_report/
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -78,4 +76,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run non-intrusive OCP 4.13 QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/qe-ocp-414-intrusive.yaml

@@ -62,12 +62,10 @@ jobs:
           sudo rm -rf /tmp/tnf_config/
           sudo rm -rf /tmp/tnf_report/
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=false ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=false ENABLE_PARALLEL=false ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -79,4 +77,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run intrusive OCP 4.14 QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/qe-ocp-414.yaml

@@ -61,12 +61,10 @@ jobs:
           sudo rm -rf /tmp/tnf_config/
           sudo rm -rf /tmp/tnf_report/
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -78,4 +76,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run non-intrusive OCP 4.14 QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/qe.yaml

@@ -10,6 +10,8 @@ on:
 env:
   QE_REPO: test-network-function/cnfcert-tests-verification
 
+
+
 jobs:
   qe-testing:
     runs-on: qe-runner
@@ -103,12 +105,10 @@ jobs:
         run: ./scripts/delete-namespaces.sh
         working-directory: cnfcert-tests-verification
 
+      # Setup is complete.  Time to run the QE tests.
       - name: Run the tests
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 60
-          max_attempts: 3
-          command: cd ${GITHUB_WORKSPACE}/cnfcert-tests-verification; FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        run: FEATURES=${{matrix.suite}} TNF_REPO_PATH=${GITHUB_WORKSPACE} TNF_IMAGE=${{env.TEST_TNF_IMAGE_NAME}} TNF_IMAGE_TAG=${{env.TEST_TNF_IMAGE_TAG}} DISABLE_INTRUSIVE_TESTS=true ENABLE_PARALLEL=true ENABLE_FLAKY_RETRY=true make test-features
+        working-directory: cnfcert-tests-verification
 
       - name: (if on main and upstream) Send chat msg to dev team if failed to run QE tests
         if: ${{ failure() && github.ref == 'refs/heads/main' && github.repository_owner == 'test-network-function' }}
@@ -120,4 +120,4 @@ jobs:
         run: |
           curl -X POST --data "{
               \"text\": \"🚨⚠️  Failed to run Kind-based non-intrusive QE tests from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.QE_NIGHTLY_WEBHOOK }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.QE_NIGHTLY_WEBHOOK }}

.github/workflows/tnf-image.yaml

@@ -144,4 +144,4 @@ jobs:
         run: |
           curl -X POST --data "{
             \"text\": \"🚨⚠️  Failed to create container image version \`$TNF_VERSION\` from commit \<$GITHUB_REPO/commit/$COMMIT_SHA|$COMMIT_SHA\>, job ID \<$GITHUB_REPO/actions/runs/$JOB_RUN_ID/attempts/$JOB_RUN_ATTEMPT|$JOB_RUN_ID\> \"
-          }" -H 'Content-type: application/json; charset=UTF-8' '${{ secrets.GCHAT_WEBHOOK_URL }}'
+          }" -H 'Content-type: application/json; charset=UTF-8' ${{ secrets.GCHAT_WEBHOOK_URL }}

CONTRIBUTING.md

@@ -71,6 +71,16 @@ following command:
 make test
 ```
 
+## Configuration guidelines
+
+Many Tests will require some form of extra configuration. To maintain reproducibility and auditability outcomes this
+configuration must be included in a claim file. For all current configuration approaches (see the `generic` test spec)
+this will be done automatically provided the `config` structure for the Test implements or inherits a working `MarshalJSON` and `UnmarshalJSON`
+interface so it can be included in a
+[test-network-function-claim](https://github.com/test-network-function/test-network-function-claim) JSON file.
+
+All configuration must adhere to these two requirements will automatically be included in the claim.
+
 ## Documentation guidelines
 
 Each exported API, global variable or constant must have proper documentation which adheres to `gofmt`.

Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi:8.9-1028 AS build
+FROM registry.access.redhat.com/ubi8/ubi:8.8-1067.1698056881 AS build
 ENV TNF_DIR=/usr/tnf
 ENV \
 	TNF_SRC_DIR=${TNF_DIR}/tnf-src \
@@ -99,7 +99,7 @@ FROM quay.io/testnetworkfunction/oct:latest AS db
 
 # Copy the state into a new flattened image to reduce size.
 # TODO run as non-root
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9-1029
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8-1072.1697626218
 
 ENV \
 	TNF_DIR=/usr/tnf \

cmd/tnf/claim/add/add.go

@@ -9,8 +9,8 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
 	"github.com/test-network-function/cnf-certification-test/pkg/junit"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 var (

cmd/tnf/claim/add/add_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 func TestReadClaim(t *testing.T) {

cmd/tnf/claim/compare/versions/versions.go

@@ -5,7 +5,7 @@ import (
 	"log"
 
 	"github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/compare/diff"
-	officialClaimScheme "github.com/test-network-function/cnf-certification-test/pkg/claim"
+	officialClaimScheme "github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 type DiffReport struct {

cmd/tnf/claim/compare/versions/versions_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/compare/diff"
-	officialClaimScheme "github.com/test-network-function/cnf-certification-test/pkg/claim"
+	officialClaimScheme "github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 func TestCompare(t *testing.T) {

cmd/tnf/claim/show/csv/csv.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim"
 	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/identifiers"
-	claimschema "github.com/test-network-function/cnf-certification-test/pkg/claim"
+	claimschema "github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 var (

cmd/tnf/generate/catalog/catalog.go

@@ -26,7 +26,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/identifiers"
 	"github.com/test-network-function/cnf-certification-test/pkg/arrayhelper"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 
 	"github.com/spf13/cobra"
 )

cmd/tnf/generate/catalog/catalog_test.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/test-network-function/cnf-certification-test/pkg/arrayhelper"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 func TestNewCommand(t *testing.T) {

cmd/tnf/generate/qe_coverage/qe_coverage.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/test-network-function/cnf-certification-test/cnf-certification-test/identifiers"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 type TestCoverageSummaryReport struct {

cmd/tnf/generate/qe_coverage/qe_coverage_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/test-network-function/cnf-certification-test/pkg/claim"
+	"github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 type catalogEntry struct {

cmd/tnf/pkg/claim/claim.go

@@ -6,7 +6,7 @@ import (
 	"os"
 
 	"github.com/Masterminds/semver/v3"
-	officialClaimScheme "github.com/test-network-function/cnf-certification-test/pkg/claim"
+	officialClaimScheme "github.com/test-network-function/test-network-function-claim/pkg/claim"
 )
 
 const (