diff --git a/.github/.env b/.github/.env new file mode 100644 index 0000000000..eae28d66a3 --- /dev/null +++ b/.github/.env @@ -0,0 +1 @@ +TEST_CLUSTER=kind diff --git a/.github/actions/action-dotenv-to-setenv b/.github/actions/action-dotenv-to-setenv new file mode 160000 index 0000000000..a6380f4d90 --- /dev/null +++ b/.github/actions/action-dotenv-to-setenv @@ -0,0 +1 @@ +Subproject commit a6380f4d905f42adb7c065db63e6d66d75b971ad diff --git a/.github/actions/e2e-build/action.yml b/.github/actions/e2e-build/action.yml new file mode 100644 index 0000000000..feaad8cde1 --- /dev/null +++ b/.github/actions/e2e-build/action.yml @@ -0,0 +1,40 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-build +description: 'End-to-End tests for build use-cases' + +runs: + using: "composite" + + steps: + + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - name: Cache modules + uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Test + shell: bash + run: make diff --git a/.github/actions/e2e-builder/action.yml b/.github/actions/e2e-builder/action.yml new file mode 100644 index 0000000000..c08d04518e --- /dev/null +++ b/.github/actions/e2e-builder/action.yml @@ -0,0 +1,103 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-builder +description: 'End-to-End tests for builder use-cases' + +inputs: + cluster-config-data: + description: 'The configuration of the underlying cluster (if cluster-type is custom)' + required: false + cluster-kube-config-data: + description: 'Base16 encoded kube config - required for custom cluster type only' + required: false + publisher: + description: 'The publishing strategy to be used' + required: true + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: config-cluster + name: Configure Platform + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: ${{ env.TEST_CLUSTER }} + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available - depends on cluster being tested + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: install-kamel-cluster-setup + name: Install Kamel Cluster Setup + uses: ./.github/actions/kamel-install-cluster-setup + with: + kube-admin-user-ctx: ${{ steps.config-cluster.outputs.cluster-kube-admin-user-ctx }} + + - id: preflight-test + name: Preflight Check Test + uses: ./.github/actions/kamel-preflight-test + with: + build-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + image-registry-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-name: ${{ steps.build-kamel.outputs.build-binary-local-image-name }} + image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} + image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: builder + + - id: run-it + name: Run IT + shell: bash + env: + KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY: ${{ inputs.publisher }} + run: | + ./.github/actions/e2e-builder/exec-tests.sh \ + -c "${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }}" \ + -i "${{ steps.config-cluster.outputs.cluster-image-namespace }}" \ + -l "${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }}" \ + -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \ + -s "${{steps.config-cluster.outputs.cluster-image-registry-insecure }}" \ + -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" + + - name: Cleanup + uses: ./.github/actions/kamel-cleanup + if: ${{ always() }} + with: + build-bundle-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} diff --git a/.github/actions/e2e-builder/exec-tests.sh b/.github/actions/e2e-builder/exec-tests.sh new file mode 100755 index 0000000000..2a20c4bcb3 --- /dev/null +++ b/.github/actions/e2e-builder/exec-tests.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the builder tests +# +#### + +set -e + +while getopts ":c:i:l:n:s:v:x:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + x) + SAVE_FAILED_TEST_NS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# +# If bundle has been built and installed then use it +# +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + export KAMEL_INSTALL_OLM_SOURCE_NAMESPACE=${IMAGE_NAMESPACE} + export KAMEL_INSTALL_OLM_SOURCE=${BUILD_CATALOG_SOURCE} +fi + +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} +export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} +export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} +export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS} + +# Then run integration tests +make test-builder diff --git a/.github/actions/e2e-knative-yaks/action.yml b/.github/actions/e2e-knative-yaks/action.yml new file mode 100644 index 0000000000..265974775d --- /dev/null +++ b/.github/actions/e2e-knative-yaks/action.yml @@ -0,0 +1,104 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-knative-yaks +description: 'End-to-End tests for knative use-cases using yaks' + +inputs: + cluster-config-data: + description: 'The configuration of the underlying cluster (if cluster-type is custom)' + required: false + cluster-kube-config-data: + description: 'Base16 encoded kube config - required for custom cluster type only' + required: false + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: config-cluster + name: Configure Platform + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: ${{ env.TEST_CLUSTER }} + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + + - name: Install YAKS + uses: ./.github/actions/kamel-install-yaks + + - name: Install Knative + uses: ./.github/actions/kamel-install-knative + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available - depends on cluster being tested + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: install-kamel-cluster-setup + name: Install Kamel Cluster Setup + uses: ./.github/actions/kamel-install-cluster-setup + with: + kube-admin-user-ctx: ${{ steps.config-cluster.outputs.cluster-kube-admin-user-ctx }} + + - id: preflight-test + name: Preflight Check Test + uses: ./.github/actions/kamel-preflight-test + with: + build-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + image-registry-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-name: ${{ steps.build-kamel.outputs.build-binary-local-image-name }} + image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} + image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: yaks/common + + - id: run-it + name: Run IT + shell: bash + run: | + ./.github/actions/e2e-knative-yaks/exec-tests.sh \ + -c "${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }}" \ + -i "${{ steps.config-cluster.outputs.cluster-image-namespace }}" \ + -l "${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }}" \ + -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \ + -s "${{steps.config-cluster.outputs.cluster-image-registry-insecure }}" \ + -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" + + - name: Cleanup + uses: ./.github/actions/kamel-cleanup + if: ${{ always() }} + with: + build-bundle-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} diff --git a/.github/actions/e2e-knative-yaks/exec-tests.sh b/.github/actions/e2e-knative-yaks/exec-tests.sh new file mode 100755 index 0000000000..428018a51a --- /dev/null +++ b/.github/actions/e2e-knative-yaks/exec-tests.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the knative-yaks tests +# +#### + +set -e + +while getopts ":c:i:l:n:s:v:x:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + x) + SAVE_FAILED_TEST_NS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# +# If bundle has been built and installed then use it +# +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + export KAMEL_INSTALL_OLM_SOURCE_NAMESPACE=${IMAGE_NAMESPACE} + export KAMEL_INSTALL_OLM_SOURCE=${BUILD_CATALOG_SOURCE} +fi + +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} +export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} +export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} +export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS} + +# Install Yaks globally +yaks install + +# Then run integration tests +yaks test e2e/yaks/common diff --git a/.github/actions/e2e-knative/action.yml b/.github/actions/e2e-knative/action.yml new file mode 100644 index 0000000000..4689d82dc4 --- /dev/null +++ b/.github/actions/e2e-knative/action.yml @@ -0,0 +1,101 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-knative +description: 'End-to-End tests for knative use-cases' + +inputs: + cluster-config-data: + description: 'The configuration of the underlying cluster (if cluster-type is custom)' + required: false + cluster-kube-config-data: + description: 'Base16 encoded kube config - required for custom cluster type only' + required: false + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: config-cluster + name: Configure Platform + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: ${{ env.TEST_CLUSTER }} + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + + - name: Install Knative + uses: ./.github/actions/kamel-install-knative + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available - depends on cluster being tested + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: install-kamel-cluster-setup + name: Install Kamel Cluster Setup + uses: ./.github/actions/kamel-install-cluster-setup + with: + kube-admin-user-ctx: ${{ steps.config-cluster.outputs.cluster-kube-admin-user-ctx }} + + - id: preflight-test + name: Preflight Check Test + uses: ./.github/actions/kamel-preflight-test + with: + build-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + image-registry-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-name: ${{ steps.build-kamel.outputs.build-binary-local-image-name }} + image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} + image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: knative + + - id: run-it + name: Run IT + shell: bash + run: | + ./.github/actions/e2e-knative/exec-tests.sh \ + -c "${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }}" \ + -i "${{ steps.config-cluster.outputs.cluster-image-namespace }}" \ + -l "${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }}" \ + -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \ + -s "${{steps.config-cluster.outputs.cluster-image-registry-insecure }}" \ + -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" + + - name: Cleanup + uses: ./.github/actions/kamel-cleanup + if: ${{ always() }} + with: + build-bundle-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} diff --git a/.github/actions/e2e-knative/exec-tests.sh b/.github/actions/e2e-knative/exec-tests.sh new file mode 100755 index 0000000000..0053807ea5 --- /dev/null +++ b/.github/actions/e2e-knative/exec-tests.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the knative tests +# +#### + +set -e + +while getopts ":c:i:l:n:s:v:x:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + x) + SAVE_FAILED_TEST_NS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# +# If bundle has been built and installed then use it +# +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + export KAMEL_INSTALL_OLM_SOURCE_NAMESPACE=${IMAGE_NAMESPACE} + export KAMEL_INSTALL_OLM_SOURCE=${BUILD_CATALOG_SOURCE} +fi + +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} +export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} +export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} +export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS} + +export KAMEL_INSTALL_OPERATOR_ENV_VARS=KAMEL_INSTALL_DEFAULT_KAMELETS=false + +# Then run integration tests +make test-knative diff --git a/.github/actions/e2e-kubernetes/action.yml b/.github/actions/e2e-kubernetes/action.yml new file mode 100644 index 0000000000..4b60301fee --- /dev/null +++ b/.github/actions/e2e-kubernetes/action.yml @@ -0,0 +1,98 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-kubernetes +description: 'End-to-End tests for 80/20 use-cases' + +inputs: + cluster-config-data: + description: 'The configuration of the underlying cluster (if cluster-type is custom)' + required: false + cluster-kube-config-data: + description: 'Base16 encoded kube config - required for custom cluster type only' + required: false + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: config-cluster + name: Configure Platform + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: ${{ env.TEST_CLUSTER }} + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available - depends on cluster being tested + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: install-kamel-cluster-setup + name: Install Kamel Cluster Setup + uses: ./.github/actions/kamel-install-cluster-setup + with: + kube-admin-user-ctx: ${{ steps.config-cluster.outputs.cluster-kube-admin-user-ctx }} + + - id: preflight-test + name: Preflight Check Test + uses: ./.github/actions/kamel-preflight-test + with: + build-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + image-registry-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-name: ${{ steps.build-kamel.outputs.build-binary-local-image-name }} + image-registry-insecure: ${{steps.config-cluster.outputs.cluster-image-registry-insecure }} + image-version: ${{ steps.build-kamel.outputs.build-binary-local-image-version }} + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: common + + - id: run-it + name: Run IT + shell: bash + run: | + ./.github/actions/e2e-kubernetes/exec-tests.sh \ + -c "${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }}" \ + -i "${{ steps.config-cluster.outputs.cluster-image-namespace }}" \ + -l "${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }}" \ + -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \ + -s "${{steps.config-cluster.outputs.cluster-image-registry-insecure }}" \ + -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" + + - name: Cleanup + uses: ./.github/actions/kamel-cleanup + if: ${{ always() }} + with: + build-bundle-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} diff --git a/.github/actions/e2e-kubernetes/exec-tests.sh b/.github/actions/e2e-kubernetes/exec-tests.sh new file mode 100755 index 0000000000..b20cf6bcea --- /dev/null +++ b/.github/actions/e2e-kubernetes/exec-tests.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the kubernetes tests +# +#### + +set -e + +while getopts ":c:i:l:n:s:v:x:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + x) + SAVE_FAILED_TEST_NS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# +# If bundle has been built and installed then use it +# +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + export KAMEL_INSTALL_OLM_SOURCE_NAMESPACE=${IMAGE_NAMESPACE} + export KAMEL_INSTALL_OLM_SOURCE=${BUILD_CATALOG_SOURCE} +fi + +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} +export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} +export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} +export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS} + +# Then run all integration tests rather than ending on first failure +set -e +exit_code=0 +make test-integration || exit_code=1 +make test-service-binding || exit_code=1 +make test-quarkus-native || exit_code=1 +make test-kustomize || exit_code=1 +set +e + +echo "Tests completed with exit code: ${exit_code}" +exit ${exit_code} diff --git a/.github/actions/e2e-local/action.yml b/.github/actions/e2e-local/action.yml new file mode 100644 index 0000000000..a2dccccc5a --- /dev/null +++ b/.github/actions/e2e-local/action.yml @@ -0,0 +1,49 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-local +description: 'End-to-End tests for local use-cases' + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + make-rules: 'build-kamel' + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: local + + - id: run-it + name: Run IT + shell: bash + run: | + # Configure staging repos + export KAMEL_LOCAL_RUN_MAVEN_REPOSITORIES=$(make get-staging-repo) + + # Then run integration tests + make test-local diff --git a/.github/actions/e2e-upgrade/action.yml b/.github/actions/e2e-upgrade/action.yml new file mode 100644 index 0000000000..93f4495e51 --- /dev/null +++ b/.github/actions/e2e-upgrade/action.yml @@ -0,0 +1,108 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: e2e-upgrade +description: 'End-to-End tests for upgrading to new version' + +inputs: + cluster-config-data: + description: 'The configuration of the underlying cluster (if cluster-type is custom)' + required: false + cluster-kube-config-data: + description: 'Base16 encoded kube config - required for custom cluster type only' + required: false + +runs: + using: "composite" + + steps: + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env + + - id: config-cluster + name: Configure Cluster + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: ${{ env.TEST_CLUSTER }} + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + require-olm: true + + - id: released-kamel-cli + name: Get Released Kamel CLI + shell: bash + run: | + export KAMEL_VERSION=$(make get-last-released-version) + curl -L https://github.com/apache/camel-k/releases/download/v${KAMEL_VERSION}/camel-k-client-${KAMEL_VERSION}-linux-64bit.tar.gz -o /tmp/kamel.tar.gz + pushd /tmp && tar -zxf kamel.tar.gz && popd > /dev/null + if [ ! -x /tmp/kamel ]; then + echo "Error: No ${KAMEL_VERSION} downloaded correctly" + exit 1 + fi + + # + # Note: cannot use GITHUB_ENV vars in same script as it was defined + # + export RELEASED_KAMEL_BINARY=/tmp/kamel-${KAMEL_VERSION} + mv /tmp/kamel ${RELEASED_KAMEL_BINARY} + if [ $? == 0 ]; then + echo "Info: Kamel version installed: $(${RELEASED_KAMEL_BINARY} version)" + echo "::set-output name=released-kamel-binary::${RELEASED_KAMEL_BINARY}" + else + echo "Error: Failed to install kamel binary ${KAMEL_VERSION}" + exit 1 + fi + + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available. + # Since configure-cluster requires OLM then this should be true + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: report-problematic + name: List Tests Marked As Problematic + uses: ./.github/actions/kamel-report-problematic + with: + test-suite: upgrade + + - name: Run IT + shell: bash + run: | + # Note different parameters due to alternative installation + ./.github/actions/e2e-upgrade/exec-tests.sh \ + -b "${{ steps.released-kamel-cli.outputs.released-kamel-binary }}" \ + -d "${{ steps.build-kamel.outputs.build-bundle-image-bundle-index }}" \ + -l "${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }}" \ + -n "${{ steps.build-kamel.outputs.build-binary-local-image-name }}" \ + -s "${{ steps.config-cluster.outputs.cluster-image-registry-insecure }}" \ + -v "${{ steps.build-kamel.outputs.build-binary-local-image-version }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" + + - name: Cleanup + uses: ./.github/actions/kamel-cleanup + if: ${{ always() }} + with: + build-bundle-catalog-source: ${{ steps.build-kamel.outputs.build-bundle-catalog-source-name }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} diff --git a/.github/actions/e2e-upgrade/exec-tests.sh b/.github/actions/e2e-upgrade/exec-tests.sh new file mode 100755 index 0000000000..78835f3397 --- /dev/null +++ b/.github/actions/e2e-upgrade/exec-tests.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the upgrade tests +# +#### + +set -e + +while getopts ":b:d:l:n:s:v:x:" opt; do + case "${opt}" in + b) + KAMEL_BINARY=${OPTARG} + ;; + d) + BUNDLE_INDEX_IMAGE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + x) + SAVE_FAILED_TEST_NS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${KAMEL_BINARY}" ]; then + echo "Error: kamel-binary not defined" + exit 1 +fi + +if [ -z "${BUNDLE_INDEX_IMAGE}" ]; then + echo "Error: bundle-index-image not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# Use the last released Kamel CLI +export RELEASED_KAMEL_BIN=${KAMEL_BINARY} + +echo "Kamel version: $(${RELEASED_KAMEL_BIN} version)" + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# Configure install options +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +# Despite building a bundle we don't want it installed immediately so no OLM_INDEX_BUNDLE var + +# Configure test options +export CAMEL_K_PREV_IIB=quay.io/operatorhubio/catalog:latest +export CAMEL_K_NEW_IIB=${BUNDLE_INDEX_IMAGE} +export CAMEL_K_PREV_UPGRADE_CHANNEL=${PREV_XY_CHANNEL} +export CAMEL_K_NEW_UPGRADE_CHANNEL=${NEW_XY_CHANNEL} +export KAMEL_K_TEST_RELEASE_VERSION=$(make get-last-released-version) +export KAMEL_K_TEST_OPERATOR_CURRENT_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} +export CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE=${SAVE_FAILED_TEST_NS} + +# Then run integration tests +make test-upgrade diff --git a/.github/actions/kamel-build-binary/action.yml b/.github/actions/kamel-build-binary/action.yml new file mode 100644 index 0000000000..86451fe109 --- /dev/null +++ b/.github/actions/kamel-build-binary/action.yml @@ -0,0 +1,56 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-build-binary +description: 'Builds kamel operator binary' + +inputs: + image-registry-push-host: + description: 'Location of image registry push host' + required: false + image-registry-pull-host: + description: 'Location of image registry pull host' + required: false + image-namespace: + description: 'Namespace in which to store the image' + required: false + default: 'apache' + make-rules: + description: 'Override the default make rules' + required: false + +runs: + using: "composite" + steps: + - id: build-operator + name: Build Kamel Operator + shell: bash + run: | + ./.github/actions/kamel-build-binary/build-binary.sh \ + -i "${{ inputs.image-namespace }}" \ + -l "${{ inputs.image-registry-pull-host }}" \ + -m "${{ inputs.make-rules }}" \ + -s "${{ inputs.image-registry-push-host }}" \ + -x "${{ env.DEBUG_USE_EXISTING_IMAGE }}" + +outputs: + build-binary-local-image-name: + description: "Full name of the binary operator image" + value: ${{ steps.build-operator.outputs.build-binary-local-image-name }} + build-binary-local-image-version: + description: "Version & tag of the binary operator image" + value: ${{ steps.build-operator.outputs.build-binary-local-image-version }} diff --git a/.github/actions/kamel-build-binary/build-binary.sh b/.github/actions/kamel-build-binary/build-binary.sh new file mode 100755 index 0000000000..76ebd2be6b --- /dev/null +++ b/.github/actions/kamel-build-binary/build-binary.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Builds the kamel binary +# +#### + +set -e + +while getopts ":i:l:m:s:x:" opt; do + case "${opt}" in + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + m) + MAKE_RULES="${OPTARG}" + ;; + s) + REGISTRY_PUSH_HOST=${OPTARG} + ;; + x) + DEBUG_USE_EXISTING_IMAGE=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -n "${REGISTRY_PUSH_HOST}" ]; then + # + # Need an image namespace if using a registry + # + if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 + fi + + # + # Build with the PUSH host to ensure the correct image:tag + # for docker to push the image. + # + export CUSTOM_IMAGE=${REGISTRY_PUSH_HOST}/${IMAGE_NAMESPACE}/camel-k +fi + +if [ -n "${DEBUG_USE_EXISTING_IMAGE}" ] && [ -n "${CUSTOM_IMAGE}" ]; then + echo "Fetching Kamel from existing build" + + docker pull ${DEBUG_USE_EXISTING_IMAGE} + id=$(docker create ${DEBUG_USE_EXISTING_IMAGE}) + docker cp $id:/usr/local/bin/kamel . + + docker tag ${DEBUG_USE_EXISTING_IMAGE} ${CUSTOM_IMAGE}:$(make get-version) + docker push ${CUSTOM_IMAGE}:$(make get-version) +else + + echo "Build Kamel from source" + + RULES="PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images" + if [ -n "${MAKE_RULES}" ]; then + RULES=" ${MAKE_RULES} " + fi + + if [ -n "${REGISTRY_PUSH_HOST}" ]; then + RULES="${RULES} images-push" + fi + + make ${RULES} +fi + +echo "Moving kamel binary to /usr/local/bin" +sudo mv ./kamel /usr/local/bin +echo "Kamel version installed: $(kamel version)" + +# +# Use the PULL host to ensure the correct image:tag +# is passed into the tests for the deployment to pull from +# +BUILD_BINARY_LOCAL_IMAGE_NAME="${REGISTRY_PULL_HOST}/${IMAGE_NAMESPACE}/camel-k" +BUILD_BINARY_LOCAL_IMAGE_VERSION="$(make get-version)" +echo "Setting build-binary-local-image-name to ${BUILD_BINARY_LOCAL_IMAGE_NAME}" +echo "::set-output name=build-binary-local-image-name::${BUILD_BINARY_LOCAL_IMAGE_NAME}" +echo "Setting build-binary-local-image-name-version to ${BUILD_BINARY_LOCAL_IMAGE_VERSION}" +echo "::set-output name=build-binary-local-image-version::${BUILD_BINARY_LOCAL_IMAGE_VERSION}" diff --git a/.github/actions/kamel-build-bundle/action.yaml b/.github/actions/kamel-build-bundle/action.yaml new file mode 100644 index 0000000000..b52ccf88e4 --- /dev/null +++ b/.github/actions/kamel-build-bundle/action.yaml @@ -0,0 +1,98 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-build-bundle +description: 'Builds kamel operator metadata bundle' + +inputs: + image-registry-push-host: + description: 'Location of image registry push host' + required: false + image-registry-pull-host: + description: 'Location of image registry pull host' + required: false + image-namespace: + description: 'Namespace in which to store the image' + required: false + default: 'apache' + local-image-name: + description: 'Reference of the camel-k image' + required: true + local-image-version: + description: "Reference of the camel-k image version" + required: true + catalog-source-namespace: + description: 'Namespace in which to install the catalog source for the bundle (if required)' + required: false + +runs: + using: "composite" + steps: + + - id: build-bundle-image + name: Build Operator bundle + shell: bash + run: | + ./.github/actions/kamel-build-bundle/build-bundle-image.sh \ + -i "${{ inputs.image-namespace }}" \ + -l "${{ inputs.image-registry-pull-host }}" \ + -n "${{ inputs.local-image-name }}" \ + -s "${{ inputs.image-registry-push-host }}" \ + -v "${{ inputs.local-image-version }}" + + - id: install-opm + name: Install opm if required + shell: bash + run: | + if ! command -v opm &> /dev/null + then + curl -L https://github.com/operator-framework/operator-registry/releases/download/v1.16.1/linux-amd64-opm -o opm + chmod +x opm + sudo mv opm /usr/local/bin/ + fi + + - id: build-index-image + name: Create New Index Image + shell: bash + run: | + ./.github/actions/kamel-build-bundle/build-index-image.sh \ + -b "${{ steps.build-bundle-image.outputs.build-bundle-local-image }}" \ + -i "${{ inputs.image-namespace }}" \ + -l "${{ inputs.image-registry-pull-host }}" \ + -n "${{ inputs.local-image-name }}" \ + -s "${{ inputs.image-registry-push-host }}" \ + -v "${{ inputs.local-image-version }}" + + - id: build-image-catalog + name: Create a new catalog to host the index image + shell: bash + run: | + ./.github/actions/kamel-build-bundle/build-image-catalog.sh \ + -c "${{ inputs.catalog-source-namespace }}" \ + -i "${{ inputs.image-namespace }}" \ + -x "${{ steps.build-index-image.outputs.build-bundle-image-bundle-index }}" + +outputs: + build-bundle-local-image: + description: "Full name of the bundle image" + value: ${{ steps.build-bundle-image.outputs.build-bundle-local-image }} + build-bundle-image-bundle-index: + description: "Full name of the bundle index image" + value: ${{ steps.build-index-image.outputs.build-bundle-image-bundle-index }} + build-bundle-catalog-source-name: + description: "Name of the catalog source for the build bundle image" + value: ${{ steps.build-image-catalog.outputs.build-bundle-catalog-source-name }} diff --git a/.github/actions/kamel-build-bundle/build-bundle-image.sh b/.github/actions/kamel-build-bundle/build-bundle-image.sh new file mode 100755 index 0000000000..ff5aef7b11 --- /dev/null +++ b/.github/actions/kamel-build-bundle/build-bundle-image.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Builds the kamel bundle image +# +#### + +set -e + +while getopts ":i:l:n:s:v:" opt; do + case "${opt}" in + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_PUSH_HOST=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +echo "Build Operator bundle" +if ! command -v kustomize &> /dev/null +then + echo "kustomize could not be found. Has it not been installed?" + exit 1 +fi + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PUSH_HOST}" ]; then + echo "Error: image-registry-push-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +# +# Using a custom single cluster can allow for use-case that old camel-k images are cached +# (see https://cloud.redhat.com/blog/image-garbage-collection-in-openshift). This is not an +# issue on ephemeral clusters like kind. +# Therefore, need to edit the bundle CSV to ensure the ImagePullPolicy of the camel-k image is +# set to "Always" to mandate that the new image is always pulled. +# +# Use kustomize to patch the deployment resource +# +pushd config/manager > /dev/null +kustomize edit add patch --path patch-image-pull-policy-always.yaml --kind Deployment +popd + +# +# Build with the PUSH host to ensure the correct image:tag +# for docker to push the image. +# +export LOCAL_IMAGE_BUNDLE=${REGISTRY_PUSH_HOST}/${IMAGE_NAMESPACE}/camel-k-bundle:${IMAGE_VERSION} +export CUSTOM_IMAGE=${IMAGE_NAME} + +export PREV_XY_CHANNEL="stable-$(make get-last-released-version | grep -Po '\d.\d')" +echo "PREV_XY_CHANNEL=${PREV_XY_CHANNEL}" >> $GITHUB_ENV +export NEW_XY_CHANNEL=stable-$(make get-version | grep -Po "\d.\d") +echo "NEW_XY_CHANNEL=${NEW_XY_CHANNEL}" >> $GITHUB_ENV + +make bundle-build \ + BUNDLE_IMAGE_NAME=${LOCAL_IMAGE_BUNDLE} \ + DEFAULT_CHANNEL="${NEW_XY_CHANNEL}" \ + CHANNELS="${NEW_XY_CHANNEL}" + +docker push ${LOCAL_IMAGE_BUNDLE} + +# +# Use the PULL host to ensure the correct image:tag +# is passed into the tests for the deployment to pull from +# +BUILD_BUNDLE_LOCAL_IMAGE="${REGISTRY_PULL_HOST}/${IMAGE_NAMESPACE}/camel-k-bundle:${IMAGE_VERSION}" +echo "Setting build-bundle-local-image to ${BUILD_BUNDLE_LOCAL_IMAGE}" +echo "::set-output name=build-bundle-local-image::${BUILD_BUNDLE_LOCAL_IMAGE}" diff --git a/.github/actions/kamel-build-bundle/build-image-catalog.sh b/.github/actions/kamel-build-bundle/build-image-catalog.sh new file mode 100755 index 0000000000..bde06339f5 --- /dev/null +++ b/.github/actions/kamel-build-bundle/build-image-catalog.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Builds the kamel bundle index image +# +#### + +set -e + +while getopts ":c:i:x:" opt; do + case "${opt}" in + c) + CATALOG_SOURCE_NAMESPACE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + x) + BUNDLE_IMAGE_INDEX=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${CATALOG_SOURCE_NAMESPACE}" ]; then + echo "No catalog source namespace defined ... skipping catalog source creation" + exit 0 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${BUNDLE_IMAGE_INDEX}" ]; then + echo "Error: build-bundle-image-bundle-index not defined" + exit 1 +fi + +kubectl get ns ${CATALOG_SOURCE_NAMESPACE} &> /dev/null +if [ $? != 0 ]; then + echo "Error: Catalog source cannot be created as namespace ${CATALOG_SOURCE_NAMESPACE} does not exist." + exit 1 +fi + +export BUILD_CATALOG_SOURCE="camel-k-test-source" +echo "Setting build-bundle-catalog-source-name to ${BUILD_CATALOG_SOURCE}" +echo "::set-output name=build-bundle-catalog-source-name::${BUILD_CATALOG_SOURCE}" + +cat < /dev/null +then + echo "opm could not be found. Has it not been installed?" + exit 1 +fi + +# Shorten the vars +PUSH_REGISTRY=${REGISTRY_PUSH_HOST} +PULL_REGISTRY=${REGISTRY_PULL_HOST} + +# +# opm requires an active pull registry from which to verify (if not download) the bundle image +# Since the image-registry-pull-host may not be visible (eg. in the case of openshift), we need +# to fake the registry to allow opm to complete its task of creating an index image. +# +# 1. Add and alias to the hosts file for the name of the image-registry +# 2. Run a container of registry:2 docker image on the same port as the image-registry (port 80 if not present) +# 3. Tag and them push the image to the registry using docker +# 4. Run opm +# + +if [ "${PULL_REGISTRY}" != "${PUSH_REGISTRY}" ]; then + # + # With the registry interfaces different then good chance that + # pull registry is not externally accessible, eg. openshift + # + + PULL_HOST=$(echo ${PULL_REGISTRY} | sed -e 's/\(.*\):.*/\1/') + PULL_PORT=$(echo ${PULL_REGISTRY} | sed -e 's/.*:\([0-9]\+\).*/\1/') + if [ -z "${PULL_PORT}" ]; then + # Use standard http port + PULL_PORT=80 + fi + + echo "Impersonating registry at ${PULL_HOST}:${PULL_PORT}" + + # + # Update both ipv4 and ipv6 addresses if they exist + # 127.0.0.1 localhost + # ::1 localhost ip6-localhost ip6-loopback + sudo sed -i "s/\(localhost.*\)/\1 ${PULL_HOST}/g" /etc/hosts + + # + # Bring up the registry:2 instance if not already started + # + reg=$(docker ps -a -q -f name=triage-registry) + if [ -n "${reg}" ]; then + docker stop triage-registry + docker rm triage-registry + fi + + docker run -d -p ${PULL_PORT}:5000 --name triage-registry registry:2 + + # + # Tag the bundle image + # + docker tag \ + ${PUSH_REGISTRY}/${IMAGE_NAMESPACE}/camel-k-bundle:${IMAGE_VERSION} \ + ${BUNDLE_IMAGE} + + # Push the bundle image to the registry + # + docker push ${BUNDLE_IMAGE} +fi + +# +# Construct an index image containing the newly built bundle image +# Bug: +# https://github.com/operator-framework/operator-registry/issues/870 +# Workaround: +# image catalog layers contain root owned files so fails with `permission denied` error. +# Running with sudo fixes this error (alternative is to switch to podman) +# +sudo opm index add \ + -c docker --skip-tls \ + --bundles ${BUNDLE_IMAGE} \ + --from-index quay.io/operatorhubio/catalog:latest \ + --tag ${LOCAL_IIB} + +docker push ${LOCAL_IIB} +BUILD_BUNDLE_LOCAL_IMAGE_BUNDLE_INDEX="${REGISTRY_PULL_HOST}/${IMAGE_NAMESPACE}/camel-k-iib:${IMAGE_VERSION}" +echo "Setting build-bundle-image-bundle-index to ${BUILD_BUNDLE_LOCAL_IMAGE_BUNDLE_INDEX}" +echo "::set-output name=build-bundle-image-bundle-index::${BUILD_BUNDLE_LOCAL_IMAGE_BUNDLE_INDEX}" diff --git a/.github/actions/kamel-build/action.yml b/.github/actions/kamel-build/action.yml new file mode 100644 index 0000000000..09de53dab0 --- /dev/null +++ b/.github/actions/kamel-build/action.yml @@ -0,0 +1,85 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-build +description: 'Builds kamel operator image and the bundle' + +inputs: + image-registry-push-host: + description: 'Location of image registry push host' + required: false + image-registry-pull-host: + description: 'Location of image registry pull host' + required: false + image-namespace: + description: 'Namespace in which to store the image' + required: false + default: 'apache' + make-rules: + description: 'Override the default make rules' + required: false + build-bundle: + description: 'Build a bundle for install into OLM catalog' + required: true + default: false + catalog-source-namespace: + description: 'Namespace in which to install the catalog source for the bundle (if required)' + required: false + +runs: + using: "composite" + steps: + - id: build-kamel-binary + name: Build Kamel Binary + uses: ./.github/actions/kamel-build-binary + with: + image-registry-push-host: ${{ inputs.image-registry-push-host }} + image-registry-pull-host: ${{ inputs.image-registry-pull-host }} + image-namespace: ${{ inputs.image-namespace }} + make-rules: ${{ inputs.make-rules }} + + # + # By default do not build the image bundle + # + - id: build-kamel-bundle + name: Build Kamel Metadata Bundle + uses: ./.github/actions/kamel-build-bundle + if: ${{ inputs.build-bundle == 'true' }} + with: + image-registry-push-host: ${{ inputs.image-registry-push-host }} + image-registry-pull-host: ${{ inputs.image-registry-pull-host }} + image-namespace: ${{ inputs.image-namespace }} + local-image-name: ${{ steps.build-kamel-binary.outputs.build-binary-local-image-name }} + local-image-version: ${{ steps.build-kamel-binary.outputs.build-binary-local-image-version }} + catalog-source-namespace: ${{ inputs.catalog-source-namespace }} + +outputs: + build-binary-local-image-name: + description: "Full name of the binary operator image" + value: ${{ steps.build-kamel-binary.outputs.build-binary-local-image-name }} + build-binary-local-image-version: + description: "Version & tag of the binary operator image" + value: ${{ steps.build-kamel-binary.outputs.build-binary-local-image-version }} + build-bundle-local-image: + description: "Full name of the bundle image" + value: ${{ steps.build-kamel-bundle.outputs.build-bundle-local-image }} + build-bundle-image-bundle-index: + description: "Full name of the bundle index image" + value: ${{ steps.build-kamel-bundle.outputs.build-bundle-image-bundle-index }} + build-bundle-catalog-source-name: + description: "Name of the catalog source for the build bundle image" + value: ${{ steps.build-kamel-bundle.outputs.build-bundle-catalog-source-name }} diff --git a/.github/actions/kamel-cleanup/action.yaml b/.github/actions/kamel-cleanup/action.yaml new file mode 100644 index 0000000000..0d6d2e1288 --- /dev/null +++ b/.github/actions/kamel-cleanup/action.yaml @@ -0,0 +1,40 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-cleanup +description: 'Cleans up the target cluster, removing any e2e test resources' + +inputs: + build-bundle-catalog-source: + description: "Name of the catalog source for the build bundle image" + required: true + image-namespace: + description: "Installed location of the images if resident on the cluster (only applies to clusters with partnered registries)" + required: false + +runs: + using: "composite" + steps: + - id: remove-installed-kamel + name: Remove Installed Kamel + shell: bash + if: ${{ always() }} + run: | + ./.github/actions/kamel-cleanup/cleanup.sh \ + -c "${{ inputs.build-bundle-catalog-source }}" \ + -i "${{ inputs.image-namespace }}" \ + -x "${{ env.CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE }}" diff --git a/.github/actions/kamel-cleanup/cleanup-knative.sh b/.github/actions/kamel-cleanup/cleanup-knative.sh new file mode 100755 index 0000000000..9b50296fc9 --- /dev/null +++ b/.github/actions/kamel-cleanup/cleanup-knative.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Perform a cleanup of knative installation +# +#### +set +e + +cleanup_resources() { + local kind="${1}" + local needle="${2}" + + # Stops "Resource not found message" and exit code to 0 + local result=$(kubectl --ignore-not-found=true get ${kind} | grep ${needle}) + if [ $? == 0 ]; then + names=$(echo "${result}" | awk '{print $1}') + for r in ${names} + do + # Timeout after 10 minutes + kubectl delete --now --timeout=600s ${kind} ${r} 1> /dev/null + done + fi + + echo "Done" +} + +# Remove any namespaces +echo -n "Removing testing knative namespaces ... " +cleanup_resources ns 'knative-eventing\|knative-serving\|kourier-system' + +# Mutating webhooks +echo -n "Removing testing knative mutating webhooks ... " +cleanup_resources MutatingWebhookConfiguration 'knative.dev' + +# Validating webhooks +echo -n "Removing testing knative validating webhooks ... " +cleanup_resources ValidatingWebhookConfiguration 'knative.dev' + +# Cluster Role Bindings +echo -n "Removing testing knative cluster role bindings ... " +cleanup_resources clusterrolebindings 'kourier\|knative\|imc' + +# Cluster Roles +echo -n "Removing testing knative cluster roles ... " +cleanup_resources clusterroles 'knative\|imc' + +# CRDS +echo -n "Removing testing knative CRDs ... " +cleanup_resources crds 'knative.dev' + +set -e diff --git a/.github/actions/kamel-cleanup/cleanup.sh b/.github/actions/kamel-cleanup/cleanup.sh new file mode 100755 index 0000000000..47e48e76dc --- /dev/null +++ b/.github/actions/kamel-cleanup/cleanup.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Perform a cleanup of the test suite +# +#### + +set -e + +while getopts ":c:i:x:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + x) + SAVE_NAMESPACES=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + + +if [ "${SAVE_NAMESPACES}" == "true" ]; then + echo "Skipping cleanup since SAVE_NAMESPACES has been set to true" + exit 0 +fi + +# +# Remove installed kamel +# +set +e +if command -v kamel &> /dev/null +then + kamel uninstall --olm=false --all +fi + +# Ensure the CRDs are removed +kubectl get crds | grep camel | awk '{print $1}' | xargs kubectl delete crd &> /dev/null +set -e + +if [ -n "${IMAGE_NAMESPACE}" ]; then + echo -n "Removing compiled image streams ... " + imgstreams="camel-k camel-k-bundle camel-k-iib" + set +e + for cis in ${imgstreams} + do + if kubectl get is ${cis} -n ${IMAGE_NAMESPACE} &> /dev/null + then + kubectl delete is ${cis} -n ${IMAGE_NAMESPACE} + fi + done + set -e + echo "Done" +fi + +# +# Remove Catalog Source +# +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + set +e + echo -n "Removing testing catalogsource ... " + kubectl get catalogsource --all-namespaces | \ + grep ${BUILD_CATALOG_SOURCE} | awk {'print $1'} | \ + xargs kubectl delete CatalogSource &> /dev/null + if [ $? == 0 ]; then + echo "Done" + else + echo + fi + + set -e +fi + +# +# Remove KNative resources +# +./.github/actions/kamel-cleanup/cleanup-knative.sh diff --git a/.github/actions/kamel-config-cluster-custom/action.yml b/.github/actions/kamel-config-cluster-custom/action.yml new file mode 100644 index 0000000000..7cb64f482c --- /dev/null +++ b/.github/actions/kamel-config-cluster-custom/action.yml @@ -0,0 +1,125 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-config-cluster-custom +description: 'Provides configuration for acessing a custom kubernetes cluster' + +inputs: + cluster-config-data: + description: 'Non-Secure Variables for the cluster configuration' + required: true + cluster-kube-config-data: + description: 'Base16 encoded kube config' + required: true + +runs: + using: "composite" + steps: + - id: connect-cluster + name: Connect to cluster + shell: bash + run: | + ./.github/actions/kamel-config-cluster-custom/connect-cluster.sh \ + -c "${{ inputs.cluster-config-data }}" \ + -k "${{ inputs.cluster-kube-config-data }}" + + - id: configure-developer-user + name: Configure Developer User + shell: bash + run: | + # Aggregate pod eviction permission to the default admin role + cat < /dev/null; then + echo 0 + else + echo 1 + fi +} + +get_property() { + VAR=$(echo "${CLUSTER_CONFIG_DATA}" | grep ${1}) + echo ${VAR#*=} +} + +if [ -z "${KUBE_CONFIG_DATA}" ]; then + echo "Error: kube config data property cannot be found" + exit 1 +fi + +if [ ! $(has_property kube-admin-user-ctx) ]; then + echo "Error: kube admin context property cannot be found" + exit 1 +fi + +if [ ! $(has_property kube-user-ctx) ]; then + echo "Error: kube user context property cannot be found" + exit 1 +fi + +if [ ! $(has_property image-registry-pull-host) ]; then + echo "Error: image registry pull host property cannot be found" + exit 1 +fi + +if [ ! $(has_property image-registry-push-host) ]; then + echo "Error: image registry build host property cannot be found" + exit 1 +fi + +echo "::set-output name=cluster-image-registry-push-host::$(get_property image-registry-push-host)" +echo "::set-output name=cluster-image-registry-pull-host::$(get_property image-registry-pull-host)" +echo "::set-output name=cluster-image-registry-insecure::$(get_property image-registry-insecure)" +echo "::set-output name=cluster-catalog-source-namespace::$(get_property catalog-source-namespace)" + +# +# Export the image namespace if defined in the cluster config +# +if [ $(has_property image-namespace) ]; then + echo "::set-output name=cluster-image-namespace::$(get_property image-namespace)" +fi + +# +# Export the context used for admin and user +# +echo "::set-output name=cluster-kube-admin-user-ctx::$(get_property kube-admin-user-ctx)" +echo "::set-output name=cluster-kube-user-ctx::$(get_property kube-user-ctx)" + +# +# Keep values private in the log +# +echo "::add-mask::$(get_property image-registry-push-host)" +echo "::add-mask::$(get_property image-registry-pull-host)" +echo "::add-mask::$(get_property kube-admin-user-ctx)" +echo "::add-mask::$(get_property kube-user-ctx)" + +# +# Export the flag for olm capability +# +echo "::set-output name=cluster-has-olm::$(get_property has-olm)" + +# +# Login to docker if registry is externally secured +# +if [ $(has_property image-registry-user) ] && [ $(has_property image-registry-token) ]; then + echo "Secured registry in use so login with docker" + docker login \ + -u $(get_property image-registry-user) \ + -p $(get_property image-registry-token) \ + $(get_property image-registry-push-host) +fi + +# Copy the kube config to the correct location for kubectl +mkdir -p $HOME/.kube +echo -n "${KUBE_CONFIG_DATA}" | base64 -d > ${HOME}/.kube/config +if [ ! -f ${HOME}/.kube/config ]; then + echo "Error: kube config file not created correctly" + exit 1 +fi + +set -e +kubectl config use-context $(get_property kube-admin-user-ctx) +if [ $? != 0 ]; then + echo "Error: Failed to select kube admin context. Is the config and context correct?" + exit 1 +fi +set +e diff --git a/.github/actions/kamel-config-cluster-kind/action.yml b/.github/actions/kamel-config-cluster-kind/action.yml new file mode 100644 index 0000000000..3034ee85ac --- /dev/null +++ b/.github/actions/kamel-config-cluster-kind/action.yml @@ -0,0 +1,71 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-config-cluster-kind +description: 'Provides configuration for making available kubernetes cluster on kind' + +runs: + using: "composite" + steps: + - id: install-cluster + name: Install Cluster + uses: container-tools/kind-action@v1 + if: ${{ env.CLUSTER_KIND_CONFIGURED != 'true' }} + with: + version: v0.11.0 + node_image: kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad + + - id: info + name: Info + shell: bash + if: ${{ env.CLUSTER_KIND_CONFIGURED != 'true' }} + run: | + kubectl cluster-info + kubectl describe nodes + + # + # Avoid bringing up the cluster repeatedly + # + echo "CLUSTER_KIND_CONFIGURED=true" >> $GITHUB_ENV + + - id: extract-config + name: Output the config + shell: bash + run: ./.github/actions/kamel-config-cluster-kind/extract-config.sh + +outputs: + cluster-image-registry-push-host: + description: "The image registry to which to push images" + value: ${{ steps.extract-config.outputs.cluster-image-registry-push-host }} + cluster-image-registry-pull-host: + description: "The image registry from which to pull images" + value: ${{ steps.extract-config.outputs.cluster-image-registry-pull-host }} + cluster-image-registry-insecure: + description: "Whether the pull registry is insecure" + value: ${{ steps.extract-config.outputs.cluster-image-registry-insecure }} + cluster-image-namespace: + description: "The namespace to install the camel-k images" + value: ${{ steps.extract-config.outputs.cluster-image-namespace }} + cluster-kube-admin-user-ctx: + description: "The context of the kube admin user" + value: ${{ steps.extract-config.outputs.cluster-kube-admin-user-ctx }} + cluster-kube-user-ctx: + description: "The context of the kube user" + value: ${{ steps.extract-config.outputs.cluster-kube-user-ctx }} + cluster-has-olm: + description: "Does the cluster have OLM" + value: ${{ steps.extract-config.outputs.cluster-has-olm }} diff --git a/.github/actions/kamel-config-cluster-kind/extract-config.sh b/.github/actions/kamel-config-cluster-kind/extract-config.sh new file mode 100755 index 0000000000..69b4924657 --- /dev/null +++ b/.github/actions/kamel-config-cluster-kind/extract-config.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Outputs the kind config to output variables +# +#### + +set -e + +# Kind has the same interface for both pushing and pulling images in its registry +echo "::set-output name=cluster-image-registry-push-host::${KIND_REGISTRY}" +echo "::set-output name=cluster-image-registry-pull-host::${KIND_REGISTRY}" +echo "::set-output name=cluster-image-registry-insecure::$(echo true)" + +# +# Export the context used for admin and user +# Since kind has no rbac switched on then these can be the same +# +echo "::set-output name=cluster-kube-admin-user-ctx::$(kubectl config current-context)" +echo "::set-output name=cluster-kube-user-ctx::$(kubectl config current-context)" + +# Set the image namespace +echo "::set-output name=cluster-image-namespace::$(echo apache)" + +# +# cluster-catalog-source-namespace intentionally blank as OLM not routinely installed +# upgrade tests will install their own catalog-source +# + +# +# Export the flag for olm capability +# +echo "::set-output name=cluster-has-olm::$(echo false)" diff --git a/.github/actions/kamel-config-cluster-ocp3/action.yml b/.github/actions/kamel-config-cluster-ocp3/action.yml new file mode 100644 index 0000000000..c6bdf96c77 --- /dev/null +++ b/.github/actions/kamel-config-cluster-ocp3/action.yml @@ -0,0 +1,229 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-config-cluster-ocp3 +description: 'Provides configuration for making available kubernetes cluster on ocp3' + +runs: + using: "composite" + steps: + - name: Get OpenShift Client (oc) + shell: bash + if: ${{ env.CLUSTER_OCP3_CONFIGURED != 'true' }} + run: | + export OPENSHIFT_VERSION=v3.11.0 + export OPENSHIFT_COMMIT=0cbc58b + export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + + sudo rm -f /etc/resolv.conf + sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf + sudo sh -c 'echo "DNS=8.8.8.8 4.4.4.4" >> /etc/systemd/resolved.conf' + sudo service systemd-resolved restart + + # set docker0 to promiscuous mode + sudo ip link set docker0 promisc on + + # Download and install the oc binary + sudo mount --make-shared / + + sudo service docker stop + sudo echo '{"insecure-registries": ["172.30.0.0/16"]}' | sudo tee /etc/docker/daemon.json > /dev/null + sudo service docker start + + DOWNLOAD_URL=https://github.com/openshift/origin/releases/download/$OPENSHIFT_VERSION/openshift-origin-client-tools-$OPENSHIFT_VERSION-$OPENSHIFT_COMMIT-linux-64bit.tar.gz + wget -O client.tar.gz ${DOWNLOAD_URL} + tar xvzOf client.tar.gz > oc.bin + sudo mv oc.bin /usr/local/bin/oc + sudo chmod 755 /usr/local/bin/oc + + - id: start-openshift + name: Start OpenShift Cluster + shell: bash + if: ${{ env.CLUSTER_OCP3_CONFIGURED != 'true' }} + run: | + # Figure out this host's IP address + IP_ADDR="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" + + # Setup cluster dir + sudo mkdir -p /home/runner/lib/oc + sudo chmod 777 /home/runner/lib/oc + cd /home/runner/lib/oc + + # Start OpenShift + oc cluster up --public-hostname=$IP_ADDR --enable=persistent-volumes,registry,router + oc login -u system:admin + + # Wait until we have a ready node in openshift + TIMEOUT=0 + TIMEOUT_COUNT=60 + until [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do + if [ -n "$(oc get nodes | grep Ready)" ]; then + break + fi + echo "openshift is not up yet" + TIMEOUT=$((TIMEOUT+1)) + sleep 5 + done + + if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then + echo "Failed to start openshift" + exit 1 + fi + + echo "openshift is deployed and reachable" + + # + # Avoid configuring the cluster repeatedly + # + echo "CLUSTER_OCP3_CONFIGURED=true" >> $GITHUB_ENV + + - id: info + name: Info + shell: bash + if: ${{ env.CLUSTER_OCP3_CONFIGURED != 'true' }} + run: | + oc describe nodes + + - id: configure-developer-user + name: Configure Developer User + shell: bash + run: | + oc login -u system:admin + + # Export the context used for admin login + echo "::set-output name=cluster-kube-admin-user-ctx::$(oc config current-context)" + + # Aggregate pod eviction permission to the default admin role + cat <> $GITHUB_ENV + else + echo "Info: No cluster configuration supplied." + echo "CLUSTER_TYPE=${{ inputs.cluster-type }}" >> $GITHUB_ENV + fi + + - id: execute-kind + name: Maybe Execute Kind Cluster + uses: ./.github/actions/kamel-config-cluster-kind + if: ${{ env.CLUSTER_TYPE == 'kind' }} + + - id: execute-ocp3 + name: Maybe Execute Minishift Cluster + uses: ./.github/actions/kamel-config-cluster-ocp3 + if: ${{ env.CLUSTER_TYPE == 'ocp3' }} + + - id: execute-custom + name: Maybe Execute Custom Cluster + uses: ./.github/actions/kamel-config-cluster-custom + if: ${{ env.CLUSTER_TYPE == 'custom' }} + with: + cluster-config-data: ${{ inputs.cluster-config-data }} + cluster-kube-config-data: ${{ inputs.cluster-kube-config-data }} + + - id: execute-invalid + name: Execute Invalid Cluster + if: ${{ env.CLUSTER_TYPE != 'kind' && env.CLUSTER_TYPE != 'ocp3' && env.CLUSTER_TYPE != 'custom' }} + shell: bash + run: | + echo "Error: Unrecognised cluster request for type of cluster. Should be kind, ocp3 or custom." + exit 1 + + - id: cluster-config + name: Extract the config from the cluster + shell: bash + run: | + case ${{ env.CLUSTER_TYPE }} in + kind) + # Does not require cluster-catalog-source-namespace + ./.github/actions/kamel-config-cluster/output-config.sh \ + -a "${{ steps.execute-kind.outputs.cluster-kube-admin-user-ctx }}" \ + -n "${{ steps.execute-kind.outputs.cluster-image-namespace }}" \ + -o "${{ steps.execute-kind.outputs.cluster-has-olm }}" \ + -p "${{ steps.execute-kind.outputs.cluster-image-registry-push-host }}" \ + -q "${{ steps.execute-kind.outputs.cluster-image-registry-pull-host }}" \ + -s "${{ steps.execute-kind.outputs.cluster-image-registry-insecure }}" \ + -u "${{ steps.execute-kind.outputs.cluster-kube-user-ctx }}" + ;; + custom) + ./.github/actions/kamel-config-cluster/output-config.sh \ + -a "${{ steps.execute-custom.outputs.cluster-kube-admin-user-ctx }}" \ + -c "${{ steps.execute-custom.outputs.cluster-catalog-source-namespace }}" \ + -n "${{ steps.execute-custom.outputs.cluster-image-namespace }}" \ + -o "${{ steps.execute-custom.outputs.cluster-has-olm }}" \ + -p "${{ steps.execute-custom.outputs.cluster-image-registry-push-host }}" \ + -q "${{ steps.execute-custom.outputs.cluster-image-registry-pull-host }}" \ + -s "${{ steps.execute-custom.outputs.cluster-image-registry-insecure }}" \ + -u "${{ steps.execute-custom.outputs.cluster-kube-user-ctx }}" + ;; + ocp3) + # Does not require cluster-catalog-source-namespace + ./.github/actions/kamel-config-cluster/output-config.sh \ + -a "${{ steps.execute-ocp3.outputs.cluster-kube-admin-user-ctx }}" \ + -n "${{ steps.execute-ocp3.outputs.cluster-image-namespace }}" \ + -o "${{ steps.execute-ocp3.outputs.cluster-has-olm }}" \ + -p "${{ steps.execute-ocp3.outputs.cluster-image-registry-push-host }}" \ + -q "${{ steps.execute-ocp3.outputs.cluster-image-registry-pull-host }}" \ + -s "${{ steps.execute-ocp3.outputs.cluster-image-registry-insecure }}" \ + -u "${{ steps.execute-ocp3.outputs.cluster-kube-user-ctx }}" + ;; + esac + + # + # Install opm if required + # + - id: install-opm + name: Install opm if required + shell: bash + run: | + if [ "${{ inputs.opm }}" == "true" ]; then + curl -L https://github.com/operator-framework/operator-registry/releases/download/v1.19.5/linux-amd64-opm -o opm + chmod +x opm + sudo mv opm /usr/local/bin/ + fi + + # + # Install OLM if required + # + - id: install-olm + name: Install OLM + shell: bash + run: | + if [ "${{ steps.cluster-config.outputs.cluster-has-olm }}" == "true" ]; then + # OLM already installed by default + echo "OLM already available in cluster" + echo "::set-output name=cluster-has-olm::${{ steps.cluster-config.outputs.cluster-has-olm }}" + exit 0 + fi + + if [ "${{ inputs.require-olm }}" != "true" ]; then + # OLM not explicitly requested + echo "OLM not explicity required for testing" + echo "::set-output name=cluster-has-olm::$(echo false)" + exit 0 + fi + + # + # Get current context + # + echo "Cache current kube context" + ctx=$(kubectl config current-context) + + # + # Need to be admin so switch to the admin context + # + echo "Change to kube admin context" + kubectl config use-context "${{ steps.cluster-config.outputs.cluster-kube-admin-user-ctx }}" + + set +e + echo "Check if OLM is already installed" + kubectl get deployments --all-namespaces | grep olm-operator + if [ $? != 0 ]; then + set -e + echo "OLM not detected on cluster so downloading and installing" + kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.17.0/crds.yaml + # wait for a while to be sure CRDs are installed + sleep 1 + kubectl apply -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.17.0/olm.yaml + fi + set -e + + # + # Change back to original context + # + echo "Return to original kube context" + kubectl config use-context "${ctx}" + + echo "::set-output name=cluster-has-olm::$(echo true)" + echo "Complete" + +outputs: + cluster-image-registry-push-host: + description: "The image registry to which to push images" + value: ${{ steps.cluster-config.outputs.cluster-image-registry-push-host }} + cluster-image-registry-pull-host: + description: "The image registry from which to pull images" + value: ${{ steps.cluster-config.outputs.cluster-image-registry-pull-host }} + cluster-image-registry-insecure: + description: "Whether the pull registry is insecure" + value: ${{ steps.cluster-config.outputs.cluster-image-registry-insecure }} + cluster-catalog-source-namespace: + description: "The namespace in which to install the OLM catalog source" + value: ${{ steps.cluster-config.outputs.cluster-catalog-source-namespace }} + cluster-image-namespace: + description: "The namespace to install the camel-k images" + value: ${{ steps.cluster-config.outputs.cluster-image-namespace }} + cluster-kube-admin-user-ctx: + description: "The context of the kube admin user" + value: ${{ steps.cluster-config.outputs.cluster-kube-admin-user-ctx }} + cluster-kube-user-ctx: + description: "The context of the kube user" + value: ${{ steps.cluster-config.outputs.cluster-kube-user-ctx }} + cluster-has-olm: + description: "Does the cluster have OLM" + value: ${{ steps.install-olm.outputs.cluster-has-olm }} diff --git a/.github/actions/kamel-config-cluster/output-config.sh b/.github/actions/kamel-config-cluster/output-config.sh new file mode 100755 index 0000000000..99e9d86d10 --- /dev/null +++ b/.github/actions/kamel-config-cluster/output-config.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Outputs the config to cluster output variables +# +#### + +set -e + +while getopts ":a:c:n:o:p:q:s:u:" opt; do + case "${opt}" in + a) + ADMIN_USER_CTX=${OPTARG} + ;; + c) + CATALOG_SOURCE_NAMESPACE=${OPTARG} + ;; + n) + IMAGE_NAMESPACE=${OPTARG} + ;; + o) + HAS_OLM=${OPTARG} + ;; + p) + PUSH_HOST=${OPTARG} + ;; + q) + PULL_HOST=${OPTARG} + ;; + s) + INSECURE=${OPTARG} + ;; + u) + USER_CTX=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +echo "::set-output name=cluster-image-registry-push-host::${PUSH_HOST}" +echo "::set-output name=cluster-image-registry-pull-host::${PULL_HOST}" +echo "::set-output name=cluster-image-registry-insecure::${INSECURE}" +echo "::set-output name=cluster-kube-admin-user-ctx::${ADMIN_USER_CTX}" +echo "::set-output name=cluster-kube-user-ctx::${USER_CTX}" + +# Set the image namespace +echo "::set-output name=cluster-image-namespace::${IMAGE_NAMESPACE}" + +# Set the catalog source namespace +echo "::set-output name=cluster-catalog-source-namespace::${CATALOG_SOURCE_NAMESPACE}" + +# +# Export the flag for olm capability +# +echo "::set-output name=cluster-has-olm::${HAS_OLM}" diff --git a/.github/actions/kamel-install-cluster-setup/action.yml b/.github/actions/kamel-install-cluster-setup/action.yml new file mode 100644 index 0000000000..4cc48fdbea --- /dev/null +++ b/.github/actions/kamel-install-cluster-setup/action.yml @@ -0,0 +1,40 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-install-cluster-setup +description: 'Execute kamel to install the cluster-level artifacts' + +inputs: + kube-admin-user-ctx: + description: "The administration user context of the cluster" + required: true + +runs: + using: "composite" + steps: + - id: execute-kamel + name: Install Camel-K Cluster Resources + shell: bash + run: | + ./.github/actions/kamel-install-cluster-setup/install-cluster-setup.sh \ + -a "${{ inputs.kube-admin-user-ctx }}" + + - id: post-execution + shell: bash + run: | + rm -f /tmp/config + export -n KUBECONFIG diff --git a/.github/actions/kamel-install-cluster-setup/install-cluster-setup.sh b/.github/actions/kamel-install-cluster-setup/install-cluster-setup.sh new file mode 100755 index 0000000000..36a706be55 --- /dev/null +++ b/.github/actions/kamel-install-cluster-setup/install-cluster-setup.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Install the kamel setup using the admin context +# +#### + +set -e + +while getopts ":a:" opt; do + case "${opt}" in + a) + KUBE_ADMIN_CTX=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${KUBE_ADMIN_CTX}" ]; then + echo "Error: kube-admin-user-ctx not defined" + exit 1 +fi + +# +# Get current context +# +ctx=$(kubectl config current-context) + +# +# Need to be admin so switch to the admin context +# +kubectl config use-context "${KUBE_ADMIN_CTX}" + +# +# Ensure built binary CRDs are always installed by turning off olm +# +kamel install --cluster-setup --olm=false + +# +# Change back to original context +# +kubectl config use-context "${ctx}" diff --git a/.github/actions/kamel-install-knative/action.yml b/.github/actions/kamel-install-knative/action.yml new file mode 100644 index 0000000000..40f647d1ef --- /dev/null +++ b/.github/actions/kamel-install-knative/action.yml @@ -0,0 +1,26 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-install-knative +description: 'Install Knative artifacts' +runs: + using: "composite" + steps: + - name: Install Knative + shell: bash + run: | + ./.github/actions/kamel-install-knative/install-knative.sh diff --git a/.github/actions/kamel-install-knative/install-knative.sh b/.github/actions/kamel-install-knative/install-knative.sh new file mode 100755 index 0000000000..8491612487 --- /dev/null +++ b/.github/actions/kamel-install-knative/install-knative.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Install the knative setup +# +#### + +set -e + +# Prerequisites +sudo wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq + +set +e + +export SERVING_VERSION=knative-v1.1.0 +export EVENTING_VERSION=knative-v1.1.0 +export KOURIER_VERSION=knative-v1.1.0 + +apply() { + local file="${1:-}" + if [ -z "${file}" ]; then + echo "Error: Cannot apply. No file." + exit 1 + fi + + kubectl apply --filename ${file} + if [ $? != 0 ]; then + sleep 5 + echo "Re-applying ${file} ..." + kubectl apply --filename ${file} + if [ $? != 0 ]; then + echo "Error: Application of resource failed." + exit 1 + fi + fi +} + +SERVING_CRDS="https://github.com/knative/serving/releases/download/${SERVING_VERSION}/serving-crds.yaml" +SERVING_CORE="https://github.com/knative/serving/releases/download/${SERVING_VERSION}/serving-core.yaml" +KOURIER="https://github.com/knative-sandbox/net-kourier/releases/download/${KOURIER_VERSION}/kourier.yaml" +EVENTING_CRDS="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/eventing-crds.yaml" +EVENTING_CORE="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/eventing-core.yaml" +IN_MEMORY_CHANNEL="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/in-memory-channel.yaml" +CHANNEL_BROKER="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/mt-channel-broker.yaml" +SUGAR_CONTROLLER="https://github.com/knative/eventing/releases/download/${EVENTING_VERSION}/eventing-sugar-controller.yaml" + +# Serving +apply "${SERVING_CRDS}" + +YAML=$(mktemp serving-core-XXX.yaml) +curl -L -s ${SERVING_CORE} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML} +if [ -s ${YAML} ]; then + apply ${YAML} + echo "Waiting for pods to be ready in knative-serving (dependency for kourier)" + kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s +else + echo "Error: Failed to correctly download ${SERVING_CORE}" + exit 1 +fi + +# Kourier +apply "${KOURIER}" + +kubectl patch configmap/config-network \ + --namespace knative-serving \ + --type merge \ + --patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}' +if [ $? != 0 ]; then + echo "Error: Failed to patch configmap" + exit 1 +fi + +# Eventing +apply "${EVENTING_CRDS}" + +YAML=$(mktemp eventing-XXX.yaml) +curl -L -s ${EVENTING_CORE} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML} +if [ -s ${YAML} ]; then + apply ${YAML} +else + echo "Error: Failed to correctly download ${SERVING_CORE}" + exit 1 +fi + +# Eventing channels +YAML=$(mktemp in-memory-XXX.yaml) +curl -L -s ${IN_MEMORY_CHANNEL} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML} +if [ -s ${YAML} ]; then + apply ${YAML} +else + echo "Error: Failed to correctly download ${SERVING_CORE}" + exit 1 +fi + +# Eventing broker +YAML=$(mktemp channel-broker-XXX.yaml) +curl -L -s ${CHANNEL_BROKER} | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - > ${YAML} +if [ -s ${YAML} ]; then + apply ${YAML} +else + echo "Error: Failed to correctly download ${SERVING_CORE}" + exit 1 +fi + +# Eventing sugar controller for injection +apply ${SUGAR_CONTROLLER} + +# Wait for installation completed +echo "Waiting for all pods to be ready in kourier-system" +kubectl wait --for=condition=Ready pod --all -n kourier-system --timeout=60s +echo "Waiting for all pods to be ready in knative-serving" +kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s +echo "Waiting for all pods to be ready in knative-eventing" +kubectl wait --for=condition=Ready pod --all -n knative-eventing --timeout=60s diff --git a/.github/actions/kamel-install-yaks/action.yml b/.github/actions/kamel-install-yaks/action.yml new file mode 100644 index 0000000000..7eb72f0cc0 --- /dev/null +++ b/.github/actions/kamel-install-yaks/action.yml @@ -0,0 +1,35 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-install-yaks +description: 'Install YAKS artifacts' +runs: + using: "composite" + steps: + - id: get-yaks + name: Get YAKS + shell: bash + run: | + export YAKS_VERSION=0.4.0 + curl --fail -L https://github.com/citrusframework/yaks/releases/download/v${YAKS_VERSION}/yaks-${YAKS_VERSION}-linux-64bit.tar.gz -o yaks.tar.gz + tar -zxf yaks.tar.gz + sudo mv yaks /usr/local/bin/ + + - name: Install YAKS + shell: bash + run: | + yaks install --cluster-setup diff --git a/.github/actions/kamel-preflight-test/action.yml b/.github/actions/kamel-preflight-test/action.yml new file mode 100644 index 0000000000..5cf1041ecd --- /dev/null +++ b/.github/actions/kamel-preflight-test/action.yml @@ -0,0 +1,55 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-preflight-test +description: 'Does a preflight-test to check the operator can be installed correctly' + +inputs: + build-catalog-source: + description: "The name of the bundle catalog (only installed in the cluster if OLM is used)" + required: false + image-namespace: + description: 'Namespace in which the image is stored' + required: true + image-registry-host: + description: 'Location of image registry host' + required: true + image-name: + description: 'Reference of the camel-k image' + required: true + image-registry-insecure: + description: "Whether the registry is insecure" + required: true + image-version: + description: "Reference of the camel-k image version" + required: true + +runs: + using: "composite" + + steps: + - id: preflight-test + name: Execute Preflight Test + shell: bash + run: | + ./.github/actions/kamel-preflight-test/preflight-test.sh \ + -c "${{ inputs.build-catalog-source }}" \ + -i "${{ inputs.image-namespace }}" \ + -l "${{ inputs.image-registry-host }}" \ + -n "${{ inputs.image-name }}" \ + -s "${{ inputs.image-registry-insecure }}" \ + -v "${{ inputs.image-version }}" diff --git a/.github/actions/kamel-preflight-test/preflight-test.sh b/.github/actions/kamel-preflight-test/preflight-test.sh new file mode 100755 index 0000000000..bacae0bceb --- /dev/null +++ b/.github/actions/kamel-preflight-test/preflight-test.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Execute the kubernetes tests +# +#### + +set -e + +while getopts ":c:i:l:n:s:v:" opt; do + case "${opt}" in + c) + BUILD_CATALOG_SOURCE=${OPTARG} + ;; + i) + IMAGE_NAMESPACE=${OPTARG} + ;; + l) + REGISTRY_PULL_HOST=${OPTARG} + ;; + n) + IMAGE_NAME=${OPTARG} + ;; + s) + REGISTRY_INSECURE=${OPTARG} + ;; + v) + IMAGE_VERSION=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${IMAGE_NAME}" ]; then + echo "Error: local-image-name not defined" + exit 1 +fi + +if [ -z "${IMAGE_VERSION}" ]; then + echo "Error: local-image-version not defined" + exit 1 +fi + +if [ -z "${IMAGE_NAMESPACE}" ]; then + echo "Error: image-namespace not defined" + exit 1 +fi + +if [ -z "${REGISTRY_PULL_HOST}" ]; then + echo "Error: image-registry-pull-host not defined" + exit 1 +fi + +if [ -z "${REGISTRY_INSECURE}" ]; then + echo "Error: image-registry-insecure not defined" + exit 1 +fi + +# +# Create the preflight test namespace +# +NAMESPACE="preflight" +set +e +if kubectl get ns ${NAMESPACE} &> /dev/null +then + kubectl delete ns ${NAMESPACE} +fi +set -e + +kubectl create namespace ${NAMESPACE} +if [ $? != 0 ]; then + echo "Error: failed to create the ${NAMESPACE} namespace" + exit 1 +fi + +#trap "kubectl delete ns ${NAMESPACE} &> /dev/null" EXIT + +# Cluster environment +export CUSTOM_IMAGE=${IMAGE_NAME} +export CUSTOM_VERSION=${IMAGE_VERSION} + +# +# If bundle has been built and installed then use it +# +has_olm="false" +if [ -n "${BUILD_CATALOG_SOURCE}" ]; then + export KAMEL_INSTALL_OLM_SOURCE_NAMESPACE=${IMAGE_NAMESPACE} + export KAMEL_INSTALL_OLM_SOURCE=${BUILD_CATALOG_SOURCE} + has_olm="true" +fi + +export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) +export KAMEL_INSTALL_REGISTRY=${REGISTRY_PULL_HOST} +export KAMEL_INSTALL_REGISTRY_INSECURE=${REGISTRY_INSECURE} +export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + +# Will only have an effect if olm=false +# since, for OLM, the csv determines the policy. +# (see kamel-build-bundle/build-bundle-image.sh) +export KAMEL_INSTALL_OPERATOR_IMAGE_PULL_POLICY="Always" + +# +# Install the operator +# +kamel install -n ${NAMESPACE} --olm=${has_olm} +if [ $? != 0 ]; then + echo "Error: kamel install returned an error." + exit 1 +fi + +sleep 3 + +# +# Wait for the operator to be running +# +timeout=180 +i=1 +command="kubectl get pods -n ${NAMESPACE} 2> /dev/null | grep camel-k | grep Running &> /dev/null" + +until eval "${command}" +do + ((i++)) + if [ "${i}" -gt "${timeout}" ]; then + echo "kamel operator not successfully installed, aborting due to ${timeout}s timeout" + exit 1 + fi + + sleep 1 +done + +echo "Camel-K operator up and running" + +camel_operator=$(kubectl get pods -n ${NAMESPACE} | grep camel-k | awk '{print $1}') +camel_op_version=$(kubectl logs ${camel_operator} -n ${NAMESPACE} | sed -n 's/.*"Camel K Operator Version: \(.*\)"}/\1/p') +camel_op_commit=$(kubectl logs ${camel_operator} -n ${NAMESPACE} | sed -n 's/.*"Camel K Git Commit: \(.*\)"}/\1/p') + +src_commit=$(git rev-parse HEAD) + +# +# Test whether the versions are the same +# +if [ "${camel_op_version}" != "${IMAGE_VERSION}" ]; then + echo "Preflight Test: Failure - Installed operator version (${camel_op_version} does not match expected version (${IMAGE_VERSION})" + exit 1 +fi + +# +# Test whether the commit ids are the same +# +if [ "${camel_op_commit}" != "${src_commit}" ]; then + echo "Preflight Test: Failure - Installed operator commit id (${camel_op_commit}) does not match expected commit id (${src_commit})" + exit 1 +fi + +echo "Preflight Test: Success" diff --git a/.github/actions/kamel-prepare-env/action.yml b/.github/actions/kamel-prepare-env/action.yml new file mode 100644 index 0000000000..45871e10b1 --- /dev/null +++ b/.github/actions/kamel-prepare-env/action.yml @@ -0,0 +1,109 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-prepare-env +description: 'Initialise the test environment with tools. (Will only run once per workflow)' + +runs: + using: "composite" + steps: + # + # This is a docker action so its pre-builds the image prior to + # running it. The building occurs prior to all steps regardless + # of where its located to run. Therefore, it must be run prior to + # clean-up since that step removes all docker image in order to + # claim back as much space as possible. + # + # Expects env file with following vars: + # TEST_CLUSTER + # + - id: read-env-file + uses: ./.github/actions/action-dotenv-to-setenv + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + with: + env-file: .github/.env + + - name: Cleanup + shell: bash + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + run: | + ls -lart + echo "Initial status:" + df -h + + if [ "$RUNNER_OS" == "Linux" ]; then + echo "Cleaning up resources:" + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + df -kh + docker rmi $(docker image ls -aq) || true # Don't fail if image is not present + df -kh + else + echo "OS $RUNNER_OS is not yet supported" + exit 1 + fi + + echo "Final status:" + df -h + + - name: Set up JDK 11 + uses: AdoptOpenJDK/install-jdk@v1 + if: ${{ env.ENV_PREPARED != 'true' }} + with: + version: "11" + + - name: Set Go + uses: actions/setup-go@v2 # Version 2 adds GOBIN to PATH + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + with: + go-version: 1.16.x + + - name: (Re-)install kustomize + shell: bash + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + run: | + # reinstall kustomize to be always on the same version + sudo rm $(which kustomize) + make kustomize + + # Add kustomize to PATH + echo "${{ env.GOPATH }}/bin" >> $GITHUB_PATH + which kustomize || { echo 'kustomize not found' ; exit 1; } + + # + # Install a version of kubectl for generic access to cluster + # + - id: install-kubectl + uses: azure/setup-kubectl@v1 + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + with: + version: 'latest' + + - id: complete-action + name: Environment Prepared + shell: bash + if: ${{ env.KAMEL_PREPARE_ENV != 'true' }} + run : | + # + # Avoid preparing the environment repeatedly + # + echo "KAMEL_PREPARE_ENV=true" >> $GITHUB_ENV diff --git a/.github/actions/kamel-report-problematic/action.yml b/.github/actions/kamel-report-problematic/action.yml new file mode 100644 index 0000000000..c981858aba --- /dev/null +++ b/.github/actions/kamel-report-problematic/action.yml @@ -0,0 +1,35 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +name: kamel-report-problematic +description: 'Reports the number of tests labelled as problematic in test suite' + +inputs: + test-suite: + description: "The test suite to be analysed" + required: true + +runs: + using: "composite" + steps: + - id: report-problematic + name: Report Tests Marked As Problematic + shell: bash + if: ${{ always() }} + run: | + ./.github/actions/kamel-report-problematic/report-problematic.sh \ + -t "${{ inputs.test-suite }}" diff --git a/.github/actions/kamel-report-problematic/report-problematic.sh b/.github/actions/kamel-report-problematic/report-problematic.sh new file mode 100755 index 0000000000..53f8837996 --- /dev/null +++ b/.github/actions/kamel-report-problematic/report-problematic.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +#### +# +# Find all test that are labelled as problematic +# +#### + +set -e + +while getopts ":t:" opt; do + case "${opt}" in + t) + TEST_SUITE=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "${TEST_SUITE}" ]; then + echo "Error: ${0} -t " + exit 1 +fi + +TEST_DIR="./e2e/${TEST_SUITE}" + +if [ ! -d "${TEST_DIR}" ]; then + echo "No e2e directory available ... exiting" + exit 0 +fi + +PROBLEMATIC=() +while IFS= read -r -d '' f +do + + func="" + while IFS= read -r line + do + if [[ "${line}" =~ ^" * " ]]; then + continue + elif [[ "${line}" =~ ^func* ]]; then + func=$(echo "${line}" | sed -n "s/func \([a-zA-Z0-9]\+\).*/\1/p") + elif [[ "${line}" =~ CAMEL_K_TEST_SKIP_PROBLEMATIC ]]; then + PROBLEMATIC[${#PROBLEMATIC[*]}]="${f}::${func}" + fi + done < ${f} + +done < <(find "${TEST_DIR}" -name "*.go" -print0) + +if [ "${CAMEL_K_TEST_SKIP_PROBLEMATIC}" == "true" ]; then + if [ ${#PROBLEMATIC[*]} -gt 0 ]; then + echo "=== Problematic Tests (${#PROBLEMATIC[*]}) ===" + for prob in "${PROBLEMATIC[@]}" + do + echo " ${prob}" + done + echo "===" + else + echo "=== No Tests marked as Problematic ===" + fi +else + echo "=== All tests to be executed, including those marked as problematic (${#PROBLEMATIC[*]}) ===" +fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7831f9e58a..217b404c5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,6 +42,8 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -54,24 +56,14 @@ jobs: # TODO: test an all the supported OS # [ubuntu-20.04, macos-latest, windows-latest] os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} steps: - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Install Go - uses: actions/setup-go@v1 - with: - go-version: 1.16.x - name: Checkout code uses: actions/checkout@v2 - - name: Cache modules - uses: actions/cache@v1 with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Test - run: make + persist-credentials: false + submodules: recursive + + - name: Execute Build Tests + uses: ./.github/actions/e2e-build diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index b520ff8a7e..1c9e5f8b34 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -42,6 +42,20 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: + pre-built-kamel-image: + description: 'Kamel image url for skipping building of kamel stages. Used for debugging' + required: false + skip-problematic: + description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' + required: false + default: false + test-filters: + description: | + Filter the tests in this test suite by assigning the test pattern to TEST_KNATIVE_RUN, + eg. TEST_KNATIVE_RUN=TestBasic will only run tests prefixed with 'TestBasic' + required: false concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -58,72 +72,21 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 - with: - go-version: 1.16.x - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 with: - version: v0.11.0 - node_image: kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad - - name: Info - run: | - kubectl cluster-info - kubectl describe nodes - - name: Build Operator - run: | - echo "Build project" - export CUSTOM_IMAGE=${KIND_REGISTRY}/apache/camel-k - echo "LOCAL_IMAGE_NAME=${CUSTOM_IMAGE}" >> $GITHUB_ENV - echo "LOCAL_IMAGE_VERSION=$(make get-version)" >> $GITHUB_ENV - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images images-push + persist-credentials: false + submodules: recursive - sudo mv ./kamel /usr/local/bin - - name: Run IT - # Disable registry tests as not compatible with KinD - #env: - # TEST_DOCKER_HUB_USERNAME: ${{ secrets.TEST_DOCKER_HUB_USERNAME }} - # TEST_DOCKER_HUB_PASSWORD: ${{ secrets.TEST_DOCKER_HUB_PASSWORD }} - # TEST_GITHUB_PACKAGES_REPO: ${{ secrets.TEST_GITHUB_PACKAGES_REPO }} - # TEST_GITHUB_PACKAGES_USERNAME: ${{ secrets.TEST_GITHUB_PACKAGES_USERNAME }} - # TEST_GITHUB_PACKAGES_PASSWORD: ${{ secrets.TEST_GITHUB_PACKAGES_PASSWORD }} - env: - KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY: ${{ matrix.publisher }} + - name: Convert input parameters to env vars + shell: bash run: | - echo "Installing camel k cluster resources" - kamel install --cluster-setup - - # Configure install options - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export CUSTOM_VERSION=${{ env.LOCAL_IMAGE_VERSION }} - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - export KAMEL_INSTALL_REGISTRY=$KIND_REGISTRY - export KAMEL_INSTALL_REGISTRY_INSECURE=true - export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} - export KAMEL_INSTALL_OPERATOR_ENV_VARS=KAMEL_INSTALL_DEFAULT_KAMELETS=false + ./.github/workflows/manual-exec-process-inputs.sh \ + -i "${{ github.event.inputs.pre-built-kamel-image }}" \ + -p "${{ github.event.inputs.skip-problematic }}" \ + -t "${{ github.event.inputs.test-filters }}" - # Configure test options - export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} - export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} - - # Then run integration tests - make test-builder + - name: Execute Builder Tests + uses: ./.github/actions/e2e-builder + with: + cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }} + cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }} + publisher: ${{ matrix.publisher }} diff --git a/.github/workflows/knative.yml b/.github/workflows/knative.yml index e333dbc1cc..0c3c5c5a50 100644 --- a/.github/workflows/knative.yml +++ b/.github/workflows/knative.yml @@ -42,6 +42,20 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: + pre-built-kamel-image: + description: 'Kamel image url for skipping building of kamel stages. Used for debugging' + required: false + skip-problematic: + description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' + required: false + default: false + test-filters: + description: | + Filter the tests in this test suite by assigning the test pattern to TEST_KNATIVE_RUN, + eg. TEST_KNATIVE_RUN=TestBasic will only run tests prefixed with 'TestBasic' + required: false concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -51,235 +65,38 @@ jobs: test: runs-on: ubuntu-20.04 steps: + - name: Checkout code uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 with: - go-version: 1.16.x - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 - with: - version: v0.11.0 - node_image: kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad - - name: Info - run: | - kubectl version - kubectl cluster-info - kubectl describe nodes - - name: Install Knative - run: | - # Prerequisites - sudo wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq - - export SERVING_VERSION=knative-v1.1.0 - export EVENTING_VERSION=knative-v1.1.0 - export KOURIER_VERSION=knative-v1.1.0 - - # Serving - kubectl apply --filename https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-crds.yaml - curl -L -s https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-core.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - + persist-credentials: false + submodules: recursive - # Kourier - kubectl apply --filename https://github.com/knative-sandbox/net-kourier/releases/download/$KOURIER_VERSION/kourier.yaml - kubectl patch configmap/config-network \ - --namespace knative-serving \ - --type merge \ - --patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}' - - # Eventing - kubectl apply --filename https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-crds.yaml - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-core.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing channels - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/in-memory-channel.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing broker - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/mt-channel-broker.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing sugar controller for injection - kubectl apply -f https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-sugar-controller.yaml - - # Wait for installation completed - echo "Waiting for all pods to be ready in kourier-system" - kubectl wait --for=condition=Ready pod --all -n kourier-system --timeout=60s - echo "Waiting for all pods to be ready in knative-serving" - kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s - echo "Waiting for all pods to be ready in knative-eventing" - kubectl wait --for=condition=Ready pod --all -n knative-eventing --timeout=60s - - - name: Build Operator - run: | - echo "Build project" - export CUSTOM_IMAGE=${KIND_REGISTRY}/apache/camel-k - echo "LOCAL_IMAGE_NAME=${CUSTOM_IMAGE}" >> $GITHUB_ENV - echo "LOCAL_IMAGE_VERSION=$(make get-version)" >> $GITHUB_ENV - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images images-push - - sudo mv ./kamel /usr/local/bin - - name: Run IT + - name: Convert input parameters to env vars + shell: bash run: | - echo "Installing camel k cluster resources" - kamel install --cluster-setup - - # Configure install options - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export CUSTOM_VERSION=${{ env.LOCAL_IMAGE_VERSION }} - export KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY=Spectrum - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - export KAMEL_INSTALL_REGISTRY=$KIND_REGISTRY - export KAMEL_INSTALL_REGISTRY_INSECURE=true - export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} - - # Configure test options - export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} - export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} + ./.github/workflows/manual-exec-process-inputs.sh \ + -i "${{ github.event.inputs.pre-built-kamel-image }}" \ + -p "${{ github.event.inputs.skip-problematic }}" \ + -t "${{ github.event.inputs.test-filters }}" - # Then run integration tests - make test-knative + - name: Execute KNative Tests + uses: ./.github/actions/e2e-knative + with: + cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }} + cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }} yaks: runs-on: ubuntu-20.04 steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 - with: - go-version: 1.16.x - - name: Get YAKS - run: | - export YAKS_VERSION=0.4.0 - curl --fail -L https://github.com/citrusframework/yaks/releases/download/v${YAKS_VERSION}/yaks-${YAKS_VERSION}-linux-64bit.tar.gz -o yaks.tar.gz - tar -zxf yaks.tar.gz - sudo mv yaks /usr/local/bin/ - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 - with: - version: v0.11.0 - node_image: kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad - - name: Info - run: | - kubectl version - kubectl cluster-info - kubectl describe nodes - - name: Install YAKS - run: | - yaks install --cluster-setup - - name: Install Knative - run: | - # Prerequisites - sudo wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq - - export SERVING_VERSION=knative-v1.1.0 - export EVENTING_VERSION=knative-v1.1.0 - export KOURIER_VERSION=knative-v1.1.0 - - # Serving - kubectl apply --filename https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-crds.yaml - curl -L -s https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-core.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Kourier - kubectl apply --filename https://github.com/knative-sandbox/net-kourier/releases/download/$KOURIER_VERSION/kourier.yaml - kubectl patch configmap/config-network \ - --namespace knative-serving \ - --type merge \ - --patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}' - - # Eventing - kubectl apply --filename https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-crds.yaml - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-core.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing channels - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/in-memory-channel.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing broker - curl -L -s https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/mt-channel-broker.yaml | head -n -1 | yq e 'del(.spec.template.spec.containers[].resources)' - | kubectl apply -f - - - # Eventing sugar controller for injection - kubectl apply -f https://github.com/knative/eventing/releases/download/$EVENTING_VERSION/eventing-sugar-controller.yaml - - # Wait for installation completed - echo "Waiting for all pods to be ready in kourier-system" - kubectl wait --for=condition=Ready pod --all -n kourier-system --timeout=60s - echo "Waiting for all pods to be ready in knative-serving" - kubectl wait --for=condition=Ready pod --all -n knative-serving --timeout=60s - echo "Waiting for all pods to be ready in knative-eventing" - kubectl wait --for=condition=Ready pod --all -n knative-eventing --timeout=60s - - - name: Build Operator - run: | - echo "Build project" - export CUSTOM_IMAGE=${KIND_REGISTRY}/apache/camel-k - echo "LOCAL_IMAGE_NAME=${CUSTOM_IMAGE}" >> $GITHUB_ENV - echo "LOCAL_IMAGE_VERSION=$(make get-version)" >> $GITHUB_ENV - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images images-push - - sudo mv ./kamel /usr/local/bin - - name: Run IT - run: | - echo "Installing camel k cluster resources" - kamel install --cluster-setup - - # Configure install options - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export CUSTOM_VERSION=${{ env.LOCAL_IMAGE_VERSION }} - export KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY=Spectrum - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - export KAMEL_INSTALL_REGISTRY=$KIND_REGISTRY - export KAMEL_INSTALL_REGISTRY_INSECURE=true - export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} - - # Configure test options - export CAMEL_K_TEST_IMAGE_NAME=$KIND_REGISTRY/apache/camel-k - export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} - - # Install Yaks globally - yaks install + - name: Checkout code + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive - # Then run integration tests - yaks test e2e/yaks/common + - name: Execute Knative YAKS Tests + uses: ./.github/actions/e2e-knative-yaks + with: + cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }} + cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }} diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml index 654d70a986..b309a2a372 100644 --- a/.github/workflows/kubernetes.yml +++ b/.github/workflows/kubernetes.yml @@ -42,6 +42,23 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: + pre-built-kamel-image: + description: 'Kamel image url for skipping building of kamel stages. Used for debugging' + required: false + skip-problematic: + description: 'Whether tests marked as problematic should be skipped - false by default (sets CAMEL_K_TEST_SKIP_PROBLEMATIC)' + required: false + default: false + test-filters: + description: | + List of comma-separated key/value pairs to filter the tests in this test suite: + TEST_INTEGRATION_COMMON_RUN, TEST_INTEGRATION_COMMON_BUILD_RUN, TEST_INTEGRATION_COMMON_CLI_RUN + TEST_INTEGRATION_COMMON_CONFIG_RUN, TEST_INTEGRATION_COMMON_LANG_RUN, TEST_INTEGRATION_COMMON_TRAITS_RUN + TEST_SERVICE_RUN, TEST_QUARKUS_RUN, TEST_KUSTOMIZE_RUN + eg. TEST_KUSTOMIZE_RUN=TestBasic will only run tests prefixed with 'TestBasic' + required: false concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -55,77 +72,20 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 - with: - go-version: 1.16.x - - name: Kubernetes KinD Cluster - uses: container-tools/kind-action@v1 with: - version: v0.11.0 - node_image: kindest/node:v1.21.1@sha256:fae9a58f17f18f06aeac9772ca8b5ac680ebbed985e266f711d936e91d113bad - - name: Info - run: | - kubectl cluster-info - kubectl describe nodes - - name: Build Operator - run: | - echo "Build project" - export CUSTOM_IMAGE=${KIND_REGISTRY}/apache/camel-k - echo "LOCAL_IMAGE_NAME=${CUSTOM_IMAGE}" >> $GITHUB_ENV - echo "LOCAL_IMAGE_VERSION=$(make get-version)" >> $GITHUB_ENV - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images images-push + persist-credentials: false + submodules: recursive - sudo mv ./kamel /usr/local/bin - - name: Run IT - # Disable registry tests as not compatible with KinD - #env: - # TEST_DOCKER_HUB_USERNAME: ${{ secrets.TEST_DOCKER_HUB_USERNAME }} - # TEST_DOCKER_HUB_PASSWORD: ${{ secrets.TEST_DOCKER_HUB_PASSWORD }} - # TEST_GITHUB_PACKAGES_REPO: ${{ secrets.TEST_GITHUB_PACKAGES_REPO }} - # TEST_GITHUB_PACKAGES_USERNAME: ${{ secrets.TEST_GITHUB_PACKAGES_USERNAME }} - # TEST_GITHUB_PACKAGES_PASSWORD: ${{ secrets.TEST_GITHUB_PACKAGES_PASSWORD }} + - name: Convert input parameters to env vars + shell: bash run: | - echo "Installing camel k cluster resources" - kamel install --cluster-setup - - # Configure install options - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export CUSTOM_VERSION=${{ env.LOCAL_IMAGE_VERSION }} - export KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY=Spectrum - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - export KAMEL_INSTALL_REGISTRY=$KIND_REGISTRY - export KAMEL_INSTALL_REGISTRY_INSECURE=true - export KAMEL_INSTALL_OPERATOR_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} + ./.github/workflows/manual-exec-process-inputs.sh \ + -i "${{ github.event.inputs.pre-built-kamel-image }}" \ + -p "${{ github.event.inputs.skip-problematic }}" \ + -t "${{ github.event.inputs.test-filters }}" - # Configure test options - export CAMEL_K_TEST_IMAGE_NAME=${CUSTOM_IMAGE} - export CAMEL_K_TEST_IMAGE_VERSION=${CUSTOM_VERSION} - - # Then run integration tests - make test-integration - make test-service-binding - make test-quarkus-native - make test-kustomize + - name: Execute Tests + uses: ./.github/actions/e2e-kubernetes + with: + cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }} + cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }} diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml index a1a0856640..392238f283 100644 --- a/.github/workflows/local.yml +++ b/.github/workflows/local.yml @@ -42,6 +42,8 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -55,37 +57,9 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 with: - go-version: 1.16.x - - name: Build Kamel - run: | - echo "Build project" - make build-kamel - sudo mv ./kamel /usr/local/bin - - name: Run IT - run: | - # Configure staging repos - export KAMEL_LOCAL_RUN_MAVEN_REPOSITORIES=$(make get-staging-repo) + persist-credentials: false + submodules: recursive - # Then run integration tests - make test-local + - name: Execute Local Tests + uses: ./.github/actions/e2e-local diff --git a/.github/workflows/manual-exec-process-inputs.sh b/.github/workflows/manual-exec-process-inputs.sh new file mode 100755 index 0000000000..186e268402 --- /dev/null +++ b/.github/workflows/manual-exec-process-inputs.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +set -e + +# +# Used to unit testing this script +# +if [ -z "$GITHUB_ENV" ]; then + GITHUB_ENV="/tmp/GITHUB_ENV" + rm -f "${GITHUB_ENV}" +fi + +while getopts ":i:p:t:" opt; do + case "${opt}" in + i) + PRE_BUILT_IMAGE=${OPTARG} + ;; + p) + SKIP_PROBLEMATIC=${OPTARG} + ;; + t) + TEST_FILTERS=${OPTARG} + ;; + :) + echo "ERROR: Option -$OPTARG requires an argument" + exit 1 + ;; + \?) + echo "ERROR: Invalid option -$OPTARG" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ -n "${PRE_BUILT_IMAGE}" ]; then + echo "DEBUG_USE_EXISTING_IMAGE=${PRE_BUILT_IMAGE}" >> $GITHUB_ENV +fi + +# +# Avoid problematic tests only if parameter set to true +# +if [ "${SKIP_PROBLEMATIC}" == "true" ]; then + echo "CAMEL_K_TEST_SKIP_PROBLEMATIC=true" >> $GITHUB_ENV +fi + +# +# Adds -run args to filter tests in test suites +# +if [ -n "${TEST_FILTERS}" ]; then + filters=($(echo ${TEST_FILTERS} | tr "," "\n")) + + #Print the split string + for filter in "${filters[@]}" + do + pair=($(echo ${filter} | tr "=" " ")) + echo "Adding run filter for ${pair[0]} tests" + echo "${pair[0]}=-run ${pair[1]}" >> $GITHUB_ENV + done +fi diff --git a/.github/workflows/openshift.yml b/.github/workflows/openshift.yml index c7cd72df94..ac8ab2c780 100644 --- a/.github/workflows/openshift.yml +++ b/.github/workflows/openshift.yml @@ -42,6 +42,8 @@ on: - 'KEYS' - 'LICENSE' - 'NOTICE' + workflow_dispatch: + inputs: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -55,198 +57,40 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Set up JDK 11 - uses: AdoptOpenJDK/install-jdk@v1 - with: - version: "11" - - name: Set Go - uses: actions/setup-go@v1 with: - go-version: 1.16.x - - name: Get OpenShift Client (oc) - run: | - export OPENSHIFT_VERSION=v3.11.0 - export OPENSHIFT_COMMIT=0cbc58b - export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - sudo rm -f /etc/resolv.conf - sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf - sudo sh -c 'echo "DNS=8.8.8.8 4.4.4.4" >> /etc/systemd/resolved.conf' - sudo service systemd-resolved restart - - # set docker0 to promiscuous mode - sudo ip link set docker0 promisc on - - # Download and install the oc binary - sudo mount --make-shared / - - sudo service docker stop - sudo echo '{"insecure-registries": ["172.30.0.0/16"]}' | sudo tee /etc/docker/daemon.json > /dev/null - sudo service docker start - - DOWNLOAD_URL=https://github.com/openshift/origin/releases/download/$OPENSHIFT_VERSION/openshift-origin-client-tools-$OPENSHIFT_VERSION-$OPENSHIFT_COMMIT-linux-64bit.tar.gz - wget -O client.tar.gz ${DOWNLOAD_URL} - tar xvzOf client.tar.gz > oc.bin - sudo mv oc.bin /usr/local/bin/oc - sudo chmod 755 /usr/local/bin/oc - - - name: Start OpenShift Cluster - run: | - # Figure out this host's IP address - IP_ADDR="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" - - # Setup cluster dir - sudo mkdir -p /home/runner/lib/oc - sudo chmod 777 /home/runner/lib/oc - cd /home/runner/lib/oc - - # Start OpenShift - oc cluster up --public-hostname=$IP_ADDR --enable=persistent-volumes,registry,router - oc login -u system:admin + persist-credentials: false + submodules: recursive - # Wait until we have a ready node in openshift - TIMEOUT=0 - TIMEOUT_COUNT=60 - until [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do - if [ -n "$(oc get nodes | grep Ready)" ]; then - break - fi - echo "openshift is not up yet" - TIMEOUT=$((TIMEOUT+1)) - sleep 5 - done + - id: prepare-env + name: Prepare Test Environment + uses: ./.github/actions/kamel-prepare-env - if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then - echo "Failed to start openshift" - exit 1 - fi + - id: config-cluster + name: Configure Platform + uses: ./.github/actions/kamel-config-cluster + with: + cluster-type: 'ocp3' - echo "openshift is deployed and reachable" + - id: build-kamel + name: Build Kamel + uses: ./.github/actions/kamel-build + with: + image-registry-push-host: ${{ steps.config-cluster.outputs.cluster-image-registry-push-host }} + image-registry-pull-host: ${{ steps.config-cluster.outputs.cluster-image-registry-pull-host }} + image-namespace: ${{ steps.config-cluster.outputs.cluster-image-namespace }} + # Builds the bundle if an OLM is available - depends on cluster being tested + build-bundle: ${{ steps.config-cluster.outputs.cluster-has-olm }} + # Can be empty and so catalog source will not be created + catalog-source-namespace: ${{ steps.config-cluster.outputs.cluster-catalog-source-namespace }} + + - id: install-kamel-cluster-setup + name: Install Kamel Cluster Setup + uses: ./.github/actions/kamel-install-cluster-setup + with: + kube-admin-user-ctx: ${{ steps.config-cluster.outputs.cluster-kube-admin-user-ctx }} - - name: Info - run: | - oc describe nodes - name: Run IT - #env: - # TEST_DOCKER_HUB_USERNAME: ${{ secrets.TEST_DOCKER_HUB_USERNAME }} - # TEST_DOCKER_HUB_PASSWORD: ${{ secrets.TEST_DOCKER_HUB_PASSWORD }} - # TEST_GITHUB_PACKAGES_REPO: ${{ secrets.TEST_GITHUB_PACKAGES_REPO }} - # TEST_GITHUB_PACKAGES_USERNAME: ${{ secrets.TEST_GITHUB_PACKAGES_USERNAME }} - # TEST_GITHUB_PACKAGES_PASSWORD: ${{ secrets.TEST_GITHUB_PACKAGES_PASSWORD }} run: | - # Compute registry parameters - echo "Build project" - - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images - - # Make the Apache Snapshots or Apache Staging repository enabled by default - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - - echo "installing camel k cluster resources" - ./kamel install --cluster-setup - - # Aggregate pod eviction permission to the default admin role - cat <> $GITHUB_ENV - echo "LOCAL_IMAGE_VERSION=$(make get-version)" >> $GITHUB_ENV - make PACKAGE_ARTIFACTS_STRATEGY=download build package-artifacts images images-push - - name: Build Operator bundle - run: | - echo "Build Operator bundle" - - # reinstall kustomize to be always on the same version - sudo rm $(which kustomize) - - make kustomize - - # replace image - $(cd config/manifests && kustomize edit set image "docker.io/apache/camel-k=${{ env.LOCAL_IMAGE_NAME }}:${{ env.LOCAL_IMAGE_VERSION }}") + persist-credentials: false + submodules: recursive - # Patch CSV with the 'replaces' field to define the upgrade graph - # Use sed as the manifest/bases file is not included in the kustomize config - BASE_VERSION=$(echo ${{ env.LOCAL_IMAGE_VERSION }} | grep -Po "\d.\d.\d") - sed -i "/ version: ${BASE_VERSION}/a \ \ replaces: camel-k-operator.v$(make get-last-released-version)" config/manifests/bases/camel-k.clusterserviceversion.yaml - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export LOCAL_IMAGE_BUNDLE=${KIND_REGISTRY}/apache/camel-k-bundle:${{ env.LOCAL_IMAGE_VERSION }} - echo "LOCAL_IMAGE_BUNDLE=${LOCAL_IMAGE_BUNDLE}" >> $GITHUB_ENV - export PREV_XY_CHANNEL=stable-$(make get-last-released-version | grep -Po "\d.\d") - echo "PREV_XY_CHANNEL=${PREV_XY_CHANNEL}" >> $GITHUB_ENV - export NEW_XY_CHANNEL=stable-$(make get-version | grep -Po "\d.\d") - echo "NEW_XY_CHANNEL=${NEW_XY_CHANNEL}" >> $GITHUB_ENV - make bundle-build BUNDLE_IMAGE_NAME=${LOCAL_IMAGE_BUNDLE} DEFAULT_CHANNEL="${NEW_XY_CHANNEL}" CHANNELS="${NEW_XY_CHANNEL}" - docker push ${LOCAL_IMAGE_BUNDLE} - - name: Create new index image + - name: Convert input parameters to env vars + shell: bash run: | - export LOCAL_IIB=${KIND_REGISTRY}/apache/camel-k-iib:${{ env.LOCAL_IMAGE_VERSION }} - echo "LOCAL_IIB=${LOCAL_IIB}" >> $GITHUB_ENV - sudo opm index add --bundles ${{ env.LOCAL_IMAGE_BUNDLE }} -c docker --from-index quay.io/operatorhubio/catalog:latest --tag ${LOCAL_IIB} --skip-tls - docker push ${LOCAL_IIB} - - name: Run IT - run: | - # Use the last released Kamel CLI - export RELEASED_KAMEL_BIN=/usr/local/bin/kamel - - # Configure install options - export CUSTOM_IMAGE=${{ env.LOCAL_IMAGE_NAME }} - export CUSTOM_VERSION=${{ env.LOCAL_IMAGE_VERSION }} - export KAMEL_INSTALL_BUILD_PUBLISH_STRATEGY=Spectrum - export KAMEL_INSTALL_MAVEN_REPOSITORIES=$(make get-staging-repo) - export KAMEL_INSTALL_REGISTRY=${KIND_REGISTRY} - export KAMEL_INSTALL_REGISTRY_INSECURE=true + ./.github/workflows/manual-exec-process-inputs.sh \ + -i "${{ github.event.inputs.pre-built-kamel-image }}" \ + -p "${{ github.event.inputs.skip-problematic }}" \ + -t "${{ github.event.inputs.test-filters }}" - # Configure test options - export CAMEL_K_PREV_IIB=quay.io/operatorhubio/catalog:latest - export CAMEL_K_NEW_IIB=${{ env.LOCAL_IIB }} - export KAMEL_K_TEST_RELEASE_VERSION=$(make get-last-released-version) - export KAMEL_K_TEST_OPERATOR_CURRENT_IMAGE=${CUSTOM_IMAGE}:${CUSTOM_VERSION} - export CAMEL_K_PREV_UPGRADE_CHANNEL=${{ env.PREV_XY_CHANNEL }} - export CAMEL_K_NEW_UPGRADE_CHANNEL=${{ env.NEW_XY_CHANNEL }} - - # Then run integration tests - make test-upgrade + - name: Execute Upgrade Tests + uses: ./.github/actions/e2e-upgrade + with: + cluster-config-data: ${{ secrets.E2E_CLUSTER_CONFIG }} + cluster-kube-config-data: ${{ secrets.E2E_KUBE_CONFIG }} diff --git a/.gitmodules b/.gitmodules index 38b2437b42..4643631c43 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,7 @@ [submodule ".github/actions/changelog"] path = .github/actions/changelog url = https://github.com/CharMixer/auto-changelog-action +[submodule ".github/actions/action-dotenv-to-setenv"] + path = .github/actions/action-dotenv-to-setenv + url = https://github.com/c-py/action-dotenv-to-setenv.git + branch = tags/v3 diff --git a/e2e/builder/build_test.go b/e2e/builder/build_test.go index aca0f6fb77..4bb195ba3a 100644 --- a/e2e/builder/build_test.go +++ b/e2e/builder/build_test.go @@ -90,7 +90,7 @@ func doKitFullBuild(t *testing.T, name string, memoryLimit string, buildTimeout } Expect(Kamel(buildKitArgs...).Execute()).To(Succeed()) - Eventually(Build(ns, name)).ShouldNot(BeNil()) + Eventually(Build(ns, name), testTimeout).ShouldNot(BeNil()) Eventually(BuildPhase(ns, name), testTimeout).Should(Equal(v1.BuildPhaseSucceeded)) Eventually(KitPhase(ns, name), testTimeout).Should(Equal(v1.IntegrationKitPhaseReady)) }) diff --git a/e2e/common/cli/dev_mode_test.go b/e2e/common/cli/dev_mode_test.go index 251d83838b..db7fd7309e 100644 --- a/e2e/common/cli/dev_mode_test.go +++ b/e2e/common/cli/dev_mode_test.go @@ -38,6 +38,18 @@ import ( ) func TestRunDevMode(t *testing.T) { + + /* + * TODO + * The changing of the yaml file constant from "string" to "magic" is not being + * picked up when deploying on OCP4 and so the test is failing. + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) diff --git a/e2e/common/cli/install_test.go b/e2e/common/cli/install_test.go index 612ab75ed4..3b4d06133a 100644 --- a/e2e/common/cli/install_test.go +++ b/e2e/common/cli/install_test.go @@ -49,7 +49,7 @@ func TestKitMainInstallation(t *testing.T) { WithNewTestNamespace(t, func(ns string) { Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) Expect(Kamel("kit", "create", "timer", "-d", "camel:timer", "-n", ns).Execute()).To(Succeed()) - Eventually(Build(ns, "timer")).ShouldNot(BeNil()) + Eventually(Build(ns, "timer"), TestTimeoutMedium).ShouldNot(BeNil()) }) } diff --git a/e2e/common/config/config_test.go b/e2e/common/config/config_test.go index f764fdc97a..24c04a2593 100644 --- a/e2e/common/config/config_test.go +++ b/e2e/common/config/config_test.go @@ -242,8 +242,8 @@ func TestRunConfigExamples(t *testing.T) { t.Run("Build time property file with precedence", func(t *testing.T) { Expect(Kamel("run", "-n", ns, "./files/build-property-file-route.groovy", "--build-property", "quarkus.application.name=my-overridden-application", "--build-property", "file:./files/quarkus.properties").Execute()).To(Succeed()) Eventually(IntegrationPodPhase(ns, "build-property-file-route"), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) - Eventually(IntegrationConditionStatus(ns, "build-property-file-route", v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue)) - Eventually(IntegrationLogs(ns, "build-property-file-route"), TestTimeoutShort).Should(ContainSubstring("my-overridden-application")) + Eventually(IntegrationConditionStatus(ns, "build-property-file-route", v1.IntegrationConditionReady), TestTimeoutMedium).Should(Equal(corev1.ConditionTrue)) + Eventually(IntegrationLogs(ns, "build-property-file-route"), TestTimeoutMedium).Should(ContainSubstring("my-overridden-application")) Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed()) }) diff --git a/e2e/common/operator_metrics_test.go b/e2e/common/operator_metrics_test.go index a3f12d1926..e1944ae69d 100644 --- a/e2e/common/operator_metrics_test.go +++ b/e2e/common/operator_metrics_test.go @@ -26,6 +26,7 @@ import ( "bytes" "fmt" "math" + "os" "testing" "time" @@ -171,6 +172,17 @@ func TestMetrics(t *testing.T) { )) }) + /* + * TODO + * The duration_seconds tests keep randomly failing on OCP4 with slightly different duration values + * May need to lessen the strict checking parameters + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + t.Run("reconciliation duration metric", func(t *testing.T) { RegisterTestingT(t) Expect(metrics).To(HaveKey("camel_k_reconciliation_duration_seconds")) diff --git a/e2e/common/tekton_test.go b/e2e/common/tekton_test.go index 2ceb1c4d35..7b893c5098 100644 --- a/e2e/common/tekton_test.go +++ b/e2e/common/tekton_test.go @@ -23,6 +23,7 @@ limitations under the License. package common import ( + "os" "testing" . "github.com/onsi/gomega" @@ -32,7 +33,23 @@ import ( // TestTektonLikeBehavior verifies that the kamel binary can be invoked from within the Camel K image. // This feature is used in Tekton pipelines. + +/* + * TODO + * Test has issues on OCP4. + * Since that cluster installs via OLM, permissions are required for the + * service account doing the install to access the OLM, eg. get:subscriptions. + * + * Need to change the service account used to execute this test as do not + * want to extend the permissions of the operator service account + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ func TestTektonLikeBehavior(t *testing.T) { + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { Expect(CreateOperatorServiceAccount(ns)).To(Succeed()) Expect(CreateOperatorRole(ns)).To(Succeed()) diff --git a/e2e/common/traits/health_test.go b/e2e/common/traits/health_test.go index ccd602ccee..d17e5a98dc 100644 --- a/e2e/common/traits/health_test.go +++ b/e2e/common/traits/health_test.go @@ -25,6 +25,7 @@ package traits import ( "encoding/json" "fmt" + "os" "testing" . "github.com/onsi/gomega" @@ -35,7 +36,17 @@ import ( v1 "github.com/apache/camel-k/pkg/apis/camel/v1" ) +/* + * TODO + * Test has issues on OCP4. See TODO comment in-test for details. + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ func TestHealthTrait(t *testing.T) { + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) @@ -74,7 +85,19 @@ func TestHealthTrait(t *testing.T) { // Check the ready condition has turned falsy Eventually(IntegrationConditionStatus(ns, "java", v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionFalse)) // And it contains details about the runtime state - Eventually(IntegrationCondition(ns, "java", v1.IntegrationConditionReady)).Should(And( + + // + // TODO + // Integration has different runtime state reporting on OCP4 + // + // lastProbeTime: null + // lastTransitionTime: "2021-12-08T20:12:14Z" + // message: 'containers with unready status: [integration]' + // reason: ContainersNotReady + // status: "False" + // type: Ready + // + Eventually(IntegrationCondition(ns, "java", v1.IntegrationConditionReady), TestTimeoutMedium).Should(And( WithTransform(IntegrationConditionReason, Equal(v1.IntegrationConditionRuntimeNotReadyReason)), WithTransform(IntegrationConditionMessage, HavePrefix(fmt.Sprintf("[Pod %s runtime is not ready: map[consumer:route1:DOWN context:UP", pod.Name))), )) diff --git a/e2e/common/traits/master_test.go b/e2e/common/traits/master_test.go index 23696a5001..08de2e2317 100644 --- a/e2e/common/traits/master_test.go +++ b/e2e/common/traits/master_test.go @@ -23,6 +23,7 @@ limitations under the License. package traits import ( + "os" "testing" "time" @@ -34,6 +35,18 @@ import ( ) func TestMasterTrait(t *testing.T) { + /* + * TODO + * The test just keeps randomly failing, either on kind or OCP4 clusters. + * The integration times out before spinning up or tests for the Magicstring + * fail. + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) diff --git a/e2e/common/traits/route_test.go b/e2e/common/traits/route_test.go index 9132e49064..aeba12aadf 100644 --- a/e2e/common/traits/route_test.go +++ b/e2e/common/traits/route_test.go @@ -34,6 +34,7 @@ import ( "math/big" rand2 "math/rand" "net/http" + "os" "testing" "time" @@ -69,7 +70,18 @@ var certPem []byte // then in this case the HTTP client validates the TLS certificate. var skipClientTLSVerification = true +/* + * TODO + * Test needs to be modified as route for unsecured http is not created on OCP. + * Already skipped when executed using Kind since cluster does not support route API. + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ func TestRunRoutes(t *testing.T) { + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { ocp, err := openshift.IsOpenShift(TestClient()) if !ocp { diff --git a/e2e/common/traits/toleration_test.go b/e2e/common/traits/toleration_test.go index a58eaaf1ff..711bee440c 100644 --- a/e2e/common/traits/toleration_test.go +++ b/e2e/common/traits/toleration_test.go @@ -23,6 +23,7 @@ limitations under the License. package traits import ( + "os" "testing" . "github.com/onsi/gomega" @@ -35,7 +36,18 @@ import ( v1 "github.com/apache/camel-k/pkg/apis/camel/v1" ) +/* + * TODO + * Test needs to be modified as taint test for java3 integration does not work on OCP. + * Already skipped when executed using Kind since that is only a single-node cluster. + * + * Adding CAMEL_K_TEST_SKIP_PROBLEMATIC env var for the moment. + */ func TestTolerationTrait(t *testing.T) { + if os.Getenv("CAMEL_K_TEST_SKIP_PROBLEMATIC") == "true" { + t.Skip("WARNING: Test marked as problematic ... skipping") + } + WithNewTestNamespace(t, func(ns string) { Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) diff --git a/e2e/support/test_support.go b/e2e/support/test_support.go index 1a69db0568..72a02405ae 100644 --- a/e2e/support/test_support.go +++ b/e2e/support/test_support.go @@ -1715,6 +1715,12 @@ func DeleteKnativeBroker(ns metav1.Object) { } func DeleteTestNamespace(t *testing.T, ns ctrl.Object) { + value, saveNS := os.LookupEnv("CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE") + if t.Failed() && saveNS && value == "true" { + t.Logf("Warning: retaining failed test project %q", ns.GetName()) + return + } + var oc bool var err error if oc, err = openshift.IsOpenShift(TestClient()); err != nil { diff --git a/e2e/upgrade/cli_upgrade_test.go b/e2e/upgrade/cli_upgrade_test.go index 65415bb3f5..f8f3eeee67 100644 --- a/e2e/upgrade/cli_upgrade_test.go +++ b/e2e/upgrade/cli_upgrade_test.go @@ -50,8 +50,8 @@ func TestOperatorUpgrade(t *testing.T) { // Set KAMEL_BIN only for this test - don't override the ENV variable for all tests Expect(os.Setenv("KAMEL_BIN", kamel)).To(Succeed()) - Expect(Kamel("install", "--olm=false", "--cluster-setup", "--force").Execute()).To(Succeed()) - Expect(Kamel("install", "--olm=false", "-n", ns).Execute()).To(Succeed()) + // Should both install the CRDs and kamel in the given namespace + Expect(Kamel("install", "--olm=false", "--force", "-n", ns).Execute()).To(Succeed()) // Check the operator pod is running Eventually(OperatorPodPhase(ns), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) diff --git a/e2e/upgrade/olm_upgrade_test.go b/e2e/upgrade/olm_upgrade_test.go index 8eea3f021e..a2ff240212 100644 --- a/e2e/upgrade/olm_upgrade_test.go +++ b/e2e/upgrade/olm_upgrade_test.go @@ -98,6 +98,7 @@ func TestOLMAutomaticUpgrade(t *testing.T) { prevCSVVersion = clusterServiceVersion(noAdditionalConditions, ns)().Spec.Version prevIPVersionPrefix = fmt.Sprintf("%d.%d", prevCSVVersion.Version.Major, prevCSVVersion.Version.Minor) + t.Logf("Using Previous CSV Version: %s", prevCSVVersion.Version.String()) // Check the operator pod is running Eventually(OperatorPodPhase(ns), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) @@ -167,6 +168,10 @@ func TestOLMAutomaticUpgrade(t *testing.T) { // Rebuild the Integration Expect(Kamel("rebuild", name, "-n", ns).Execute()).To(Succeed()) + // Check the Integration runs correctly + Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutMedium).Should(Equal(corev1.ConditionTrue)) + // Check the Integration version has been upgraded Eventually(IntegrationVersion(ns, name)).Should(ContainSubstring(newIPVersionPrefix)) diff --git a/e2e/yaks/common/apache-kamelet-catalog/yaks-config.yaml b/e2e/yaks/common/apache-kamelet-catalog/yaks-config.yaml index ab0c02e724..bbac09350b 100644 --- a/e2e/yaks/common/apache-kamelet-catalog/yaks-config.yaml +++ b/e2e/yaks/common/apache-kamelet-catalog/yaks-config.yaml @@ -21,6 +21,8 @@ config: pre: - name: installation run: | - kamel install -n $YAKS_NAMESPACE + kamel install -w -n $YAKS_NAMESPACE + + kubectl wait kamelets.camel.apache.org/timer-source --for=condition=Ready --timeout=120s -n $YAKS_NAMESPACE kamel run logger.groovy -w -n $YAKS_NAMESPACE diff --git a/install/Makefile b/install/Makefile index bbc5a911ab..faf8466d6b 100644 --- a/install/Makefile +++ b/install/Makefile @@ -59,6 +59,9 @@ OPERATOR := operator PLACEHOLDER := placeholder YAML := yaml +# Fetch the latest image name - may override the original constant +IMAGE_NAME := $(shell grep image: $(MANAGER)/operator-deployment.yaml | sed 's/.*image: \(.*\):.*/\1/') + # Setup patches ROLE_TO_CROLE_PATCH := $(RBAC)/patch-role-to-clusterrole ROLEBIN_TO_CROLEBIN_PATCH := $(RBAC)/patch-rolebinding-to-clusterrolebinding @@ -77,6 +80,7 @@ INT_PLATFORM_PATCH := patch-integration-platform # define set-kustomize-image $(if $(filter $(IMAGE_NAME),$(CUSTOM_IMAGE):$(CUSTOM_VERSION)),,\ + @cd $(1) && $(KUSTOMIZE) edit set image $(IMAGE_NAME)=$(CUSTOM_IMAGE):$(CUSTOM_VERSION)) endef diff --git a/pkg/util/olm/operator.go b/pkg/util/olm/operator.go index a54a8a755f..295e988143 100644 --- a/pkg/util/olm/operator.go +++ b/pkg/util/olm/operator.go @@ -42,7 +42,7 @@ import ( // The following properties can be overridden at build time via ldflags // DefaultOperatorName is the Camel K operator name in OLM. -var DefaultOperatorName = "camel-k-operator" +var DefaultOperatorName = "camel-k" // DefaultPackage is the Camel K package in OLM. var DefaultPackage = "camel-k" diff --git a/script/Makefile b/script/Makefile index 038e15bb33..461815cbcc 100644 --- a/script/Makefile +++ b/script/Makefile @@ -16,6 +16,7 @@ VERSIONFILE := pkg/util/defaults/defaults.go VERSION ?= 1.9.0-SNAPSHOT OPERATOR_VERSION := $(subst -SNAPSHOT,,$(VERSION)) +LAST_RELEASED_IMAGE_NAME := camel-k-operator LAST_RELEASED_VERSION := 1.8.0 RUNTIME_VERSION := 1.11.0 BUILDAH_VERSION := 1.14.0 @@ -50,6 +51,12 @@ LINT_DEADLINE := 10m CHANNELS ?= $(OPERATOR_VERSION) DEFAULT_CHANNEL ?= $(OPERATOR_VERSION) PACKAGE := camel-k +CSV_VERSION := $(CUSTOM_VERSION:-SNAPSHOT=) +CSV_DISPLAY_NAME := Camel K Operator +CSV_SUPPORT := Camel +CSV_REPLACES := $(LAST_RELEASED_IMAGE_NAME).v$(LAST_RELEASED_VERSION) +CSV_FILENAME := $(PACKAGE).clusterserviceversion.yaml +CSV_PATH := config/manifests/bases/$(CSV_FILENAME) # Used to push pre-release artifacts STAGING_IMAGE_NAME := docker.io/camelk/camel-k @@ -187,44 +194,44 @@ test: build test-integration: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/common -tags=integration && \ - go test -timeout 60m -v ./e2e/common/build -tags=integration && \ - go test -timeout 60m -v ./e2e/common/cli -tags=integration && \ - go test -timeout 60m -v ./e2e/common/config -tags=integration && \ - go test -timeout 60m -v ./e2e/common/languages -tags=integration && \ - go test -timeout 60m -v ./e2e/common/traits -tags=integration + go test -timeout 60m -v ./e2e/common -tags=integration $(TEST_INTEGRATION_COMMON_RUN) && \ + go test -timeout 60m -v ./e2e/common/build -tags=integration $(TEST_INTEGRATION_COMMON_BUILD_RUN) && \ + go test -timeout 60m -v ./e2e/common/cli -tags=integration $(TEST_INTEGRATION_COMMON_CLI_RUN) && \ + go test -timeout 60m -v ./e2e/common/config -tags=integration $(TEST_INTEGRATION_COMMON_CONFIG_RUN) && \ + go test -timeout 60m -v ./e2e/common/languages -tags=integration $(TEST_INTEGRATION_COMMON_LANG_RUN) && \ + go test -timeout 60m -v ./e2e/common/traits -tags=integration $(TEST_INTEGRATION_COMMON_TRAITS_RUN) test-knative: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/knative -tags=integration + go test -timeout 60m -v ./e2e/knative -tags=integration $(TEST_KNATIVE_RUN) test-builder: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/builder -tags=integration + go test -timeout 60m -v ./e2e/builder -tags=integration $(TEST_BUILDER_RUN) test-local: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/local -tags=integration + go test -timeout 60m -v ./e2e/local -tags=integration $(TEST_LOCAL_RUN) test-kamel-cli: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/common/cli -tags=integration + go test -timeout 60m -v ./e2e/common/cli -tags=integration $(TEST_KAMEL_CLI) test-kustomize: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/common/kustomize -tags=integration + go test -timeout 60m -v ./e2e/common/kustomize -tags=integration $(TEST_KUSTOMIZE_RUN) test-quarkus-native: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/native -tags=integration + go test -timeout 60m -v ./e2e/native -tags=integration $(TEST_QUARKUS_RUN) test-service-binding: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/service-binding -tags=integration + go test -timeout 60m -v ./e2e/service-binding -tags=integration $(TEST_SERVICE_RUN) test-upgrade: build STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)" \ - go test -timeout 60m -v ./e2e/upgrade -tags=integration + go test -timeout 60m -v ./e2e/upgrade -tags=integration $(TEST_UPGRADE_RUN) build-kamel: # Ensure the binary is statically linked when building on Linux due to ABI changes in newer glibc 2.32, otherwise @@ -319,9 +326,15 @@ images-push-staging: docker tag $(CUSTOM_IMAGE):$(CUSTOM_VERSION) $(STAGING_IMAGE_NAME):$(CUSTOM_VERSION) docker push $(STAGING_IMAGE_NAME):$(CUSTOM_VERSION) +get-image: + @echo $(CUSTOM_IMAGE) + get-version: @echo $(CUSTOM_VERSION) +get-last-released-img-name: + @echo $(LAST_RELEASED_IMAGE_NAME) + get-last-released-version: @echo $(LAST_RELEASED_VERSION) @@ -335,7 +348,7 @@ git-tag: ./script/git_tag.sh $(CUSTOM_VERSION) $(RELEASE_GIT_REMOTE) cross-compile: - # we must wrap the goldflags parameters with quotes as they will need to + # we must wrap the goldflags parameters with quotes as they will need to # be processed as a single argument by the cross compile script ./script/cross_compile.sh $(CUSTOM_VERSION) $(subst ","\",$(GOFLAGS)) @@ -461,7 +474,22 @@ BUNDLE_PACKAGE := --package=$(PACKAGE) endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) $(BUNDLE_PACKAGE) -bundle: set-version generate-crd $(BUNDLE_CAMEL_APIS) kustomize operator-sdk +# +# Tailor the manifest according to default values for this project +# Note. to successfully make the bundle the name must match that specified in the PROJECT file +# +pre-bundle: +# bundle name must match that which appears in PROJECT file + @sed -i 's/projectName: .*/projectName: $(PACKAGE)/' PROJECT + @sed -i 's~^ containerImage: .*~ containerImage: $(CUSTOM_IMAGE):$(CUSTOM_VERSION)~' $(CSV_PATH) + @sed -i 's/^ support: .*/ support: $(CSV_SUPPORT)/' $(CSV_PATH) + @sed -i 's/^ name: .*.\(v.*\)/ name: $(PACKAGE).v$(CSV_VERSION)/' $(CSV_PATH) + @sed -i 's/^ displayName: .*/ displayName: $(CSV_DISPLAY_NAME)/' $(CSV_PATH) + @sed -i 's/^ replaces: .*/ replaces: $(CSV_REPLACES)/' $(CSV_PATH) + @sed -i 's/^ version: .*/ version: $(CSV_VERSION)/' $(CSV_PATH) + @sed -i '/ version: ${CSV_VERSION}/a \ \ replaces: $(CSV_REPLACES)' $(CSV_PATH) + +bundle: set-version generate-crd $(BUNDLE_CAMEL_APIS) kustomize operator-sdk pre-bundle @# Sets the operator image to the preferred image:tag @cd config/manifests && $(KUSTOMIZE) edit set image $(IMAGE_NAME)=$(CUSTOM_IMAGE):$(CUSTOM_VERSION) @# Build kustomize manifests