diff --git a/ci-operator/config/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main.yaml b/ci-operator/config/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main.yaml index a30c8d7283b64..517770e034ada 100644 --- a/ci-operator/config/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main.yaml +++ b/ci-operator/config/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main.yaml @@ -113,6 +113,15 @@ tests: steps: cluster_profile: aws-2 workflow: openshift-upgrade-aws +- always_run: false + as: e2e-aws-ovn-techpreview + optional: true + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ + steps: + cluster_profile: aws + env: + FEATURE_SET: TechPreviewNoUpgrade + workflow: openshift-e2e-aws-ccm-techpreview - as: e2e-aws-ovn-cgroupsv2 optional: true skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ diff --git a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main.yaml b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main.yaml index cad0736451952..dc85ecba663ea 100644 --- a/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main.yaml +++ b/ci-operator/config/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main.yaml @@ -88,6 +88,15 @@ tests: steps: cluster_profile: aws-2 workflow: openshift-upgrade-aws +- always_run: false + as: e2e-aws-ovn-techpreview + optional: true + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ + steps: + cluster_profile: aws + env: + FEATURE_SET: TechPreviewNoUpgrade + workflow: openshift-e2e-aws-ccm-techpreview - always_run: false as: e2e-azure-manual-oidc optional: true diff --git a/ci-operator/config/openshift/release/openshift-release-master__nightly-4.21.yaml b/ci-operator/config/openshift/release/openshift-release-master__nightly-4.21.yaml index dcb87fc74e05c..d56916d86d023 100644 --- a/ci-operator/config/openshift/release/openshift-release-master__nightly-4.21.yaml +++ b/ci-operator/config/openshift/release/openshift-release-master__nightly-4.21.yaml @@ -167,6 +167,13 @@ tests: enable: - observers-resource-watch workflow: openshift-e2e-aws-single-node +- as: e2e-aws-ccm-techpreview + interval: 168h + steps: + cluster_profile: aws + env: + FEATURE_SET: TechPreviewNoUpgrade + workflow: openshift-e2e-aws-ccm-techpreview - as: e2e-metal-ovn-single-node-rt-upgrade interval: 168h steps: diff --git a/ci-operator/jobs/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main-presubmits.yaml b/ci-operator/jobs/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main-presubmits.yaml index cc44ef14d51be..7b47304ba03b9 100644 --- a/ci-operator/jobs/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/cloud-provider-aws/openshift-cloud-provider-aws-main-presubmits.yaml @@ -149,6 +149,81 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-ovn-cgroupsv2,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build03 + context: ci/prow/e2e-aws-ovn-techpreview + decorate: true + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-cloud-provider-aws-main-e2e-aws-ovn-techpreview + optional: true + path_alias: k8s.io/cloud-provider-aws + rerun_command: /test e2e-aws-ovn-techpreview + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-ovn-techpreview + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-aws-ovn-techpreview,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main-presubmits.yaml b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main-presubmits.yaml index e7a793ea4c2d8..43d69100cecb9 100644 --- a/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/cluster-cloud-controller-manager-operator/openshift-cluster-cloud-controller-manager-operator-main-presubmits.yaml @@ -73,6 +73,80 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws-ovn,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build01 + context: ci/prow/e2e-aws-ovn-techpreview + decorate: true + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-cluster-cloud-controller-manager-operator-main-e2e-aws-ovn-techpreview + optional: true + rerun_command: /test e2e-aws-ovn-techpreview + skip_if_only_changed: ^docs/|\.md$|^(?:.*/)?(?:\.gitignore|OWNERS|PROJECT|LICENSE)$ + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-ovn-techpreview + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-aws-ovn-techpreview,?($|\s.*) - agent: kubernetes always_run: false branches: diff --git a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml index 754d240f7f6fd..34f99aa538a1d 100644 --- a/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml +++ b/ci-operator/jobs/openshift/release/openshift-release-master-periodics.yaml @@ -161167,6 +161167,82 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build11 + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: openshift + repo: release + interval: 168h + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws + ci-operator.openshift.io/variant: nightly-4.21 + ci.openshift.io/generator: prowgen + ci.openshift.io/no-builds: "true" + job-release: "4.21" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-release-master-nightly-4.21-e2e-aws-ccm-techpreview + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-ccm-techpreview + - --variant=nightly-4.21 + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build11 decorate: true diff --git a/ci-operator/step-registry/ccm/gather/OWNERS b/ci-operator/step-registry/ccm/gather/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/ccm/gather/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/ccm/gather/service-aws/OWNERS b/ci-operator/step-registry/ccm/gather/service-aws/OWNERS new file mode 120000 index 0000000000000..ec405d65a79df --- /dev/null +++ b/ci-operator/step-registry/ccm/gather/service-aws/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-commands.sh b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-commands.sh new file mode 100644 index 0000000000000..ac6c3b6da941d --- /dev/null +++ b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-commands.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# +# ccm-gather-service-aws step collects Load Balancer information from AWS API. +# + +set -o nounset +set -o errexit +set -o pipefail + +if test ! -f "${KUBECONFIG}" +then + echo "No kubeconfig, so no point in calling ccm-gather-service-aws." + exit 0 +fi + +if ! command -v aws &>/dev/null; then + echo "AWS CLI not found, skipping..." + exit 0 +fi + +if test ! -f "${CLUSTER_PROFILE_DIR}/.awscred"; then + echo "No AWS credentials, skipping..." + exit 0 +fi + +export AWS_SHARED_CREDENTIALS_FILE=${CLUSTER_PROFILE_DIR}/.awscred +export AWS_REGION=${LEASED_RESOURCE} + +function gather_lb_info_for_service() { + local service_name=$1 + local namespace=$2 + local artifact_file="${ARTIFACT_DIR}/${namespace}-${service_name}-loadbalancer.json" + + echo "Gathering Service LoadBalancer Hostname of ${namespace}/${service_name} service..." + + # Extracts the LB name inferred from the DNS Name of the service. + # AWS standard format is -..elb.amazonaws.com for Classic Load Balancers, + # and -.elb..amazonaws.com for Network Load Balancers. Examples: + # For LB_DNS=ad8c6af0820cc462c90934cd3545b5db-3a0f596def7c2ca6.elb.us-east-1.amazonaws.com, LB_NAME=ad8c6af0820cc462c90934cd3545b5db + # For LB_DNS=a3e99bc98a38549e29a699c5f9079bc9-1408355316.us-east-1.elb.amazonaws.com, LB_NAME=a3e99bc98a38549e29a699c5f9079bc9 + # For LB_DNS=mrb-v46-4vp9c-ext-6c9f52b5e9195fd4.elb.us-east-1.amazonaws.com, LB_NAME=mrb-v46-4vp9c-ext + LB_DNS=$(oc get svc/"${service_name}" -n "${namespace}" -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + LB_NAME=$(echo $LB_DNS | sed -e 's/\..*//' -e 's/-[^-]*$//') + + echo "Service (${namespace}/${service_name}): LoadBalancer Name=${LB_NAME} Hostname=${LB_DNS}" + + # CLB lookup + { + if aws elb describe-load-balancers --load-balancer-names $LB_NAME \ + --query 'LoadBalancerDescriptions[0].{LoadBalancerName:LoadBalancerName,DNSName:DNSName,CreatedTime:CreatedTime,AvailabilityZones:AvailabilityZones,SecurityGroups:SecurityGroups,IpAddressType:IpAddressType,Scheme:Scheme}' \ + --output json | jq '. + {Type: "Classic"}' > /tmp/output_clb.json; then + echo "CLB found for LoadBalancer Name=${LB_NAME}, saving to ${artifact_file}" + cat /tmp/output_clb.json >> "${artifact_file}" + else + echo "No CLB found for LoadBalancer Name=${LB_NAME}" + fi + } || true + + # NLB lookup + { + if aws elbv2 describe-load-balancers --names $LB_NAME \ + --query 'LoadBalancers[0].{DNSName:DNSName,CreatedTime:CreatedTime,LoadBalancerName:LoadBalancerName,State:State,Type:Type,AvailabilityZones:AvailabilityZones,SecurityGroups:SecurityGroups,IpAddressType:IpAddressType,Scheme:Scheme}' \ + > /tmp/output_nlb.json; then + echo "NLB found for LoadBalancer Name=${LB_NAME}, saving to ${artifact_file}" + cat /tmp/output_nlb.json >> "${artifact_file}" + else + echo "No NLB found for LoadBalancer Name=${LB_NAME}" + fi + } || true +} + +# Discovery the Load Balancer hostname of default ingresscontroller service and +# remove the domain from the hostname and AWS-appended random string to get the load balancer name +gather_lb_info_for_service "router-default" "openshift-ingress" \ No newline at end of file diff --git a/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.metadata.json b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.metadata.json new file mode 100644 index 0000000000000..dd437ec0fbffb --- /dev/null +++ b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "ccm/gather/service-aws/ccm-gather-service-aws-ref.yaml", + "owners": { + "approvers": [ + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "stephenfin" + ], + "reviewers": [ + "elmiko", + "fedosin", + "joelspeed", + "mandre", + "mdbooth", + "stephenfin" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.yaml b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.yaml new file mode 100644 index 0000000000000..21747348314b2 --- /dev/null +++ b/ci-operator/step-registry/ccm/gather/service-aws/ccm-gather-service-aws-ref.yaml @@ -0,0 +1,11 @@ +ref: + as: ccm-gather-service-aws + #optional_on_success: true + from: upi-installer + commands: ccm-gather-service-aws-commands.sh + resources: + requests: + cpu: 300m + memory: 300Mi + documentation: |- + The ccm-gather-service-aws step runs the additional step to collect Load Balancer information from AWS API. diff --git a/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/OWNERS b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/OWNERS new file mode 100644 index 0000000000000..dc1748672d6f2 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/OWNERS @@ -0,0 +1,10 @@ +approvers: +- damdo +- elmiko +- joelspeed +- nrb +reviewers: +- damdo +- elmiko +- joelspeed +- nrb \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.metadata.json b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.metadata.json new file mode 100644 index 0000000000000..144de19bd3d91 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.yaml", + "owners": { + "approvers": [ + "damdo", + "elmiko", + "joelspeed", + "nrb" + ], + "reviewers": [ + "damdo", + "elmiko", + "joelspeed", + "nrb" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.yaml b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.yaml new file mode 100644 index 0000000000000..81b307f466477 --- /dev/null +++ b/ci-operator/step-registry/openshift/e2e/aws/ccm/techpreview/openshift-e2e-aws-ccm-techpreview-workflow.yaml @@ -0,0 +1,27 @@ +workflow: + as: openshift-e2e-aws-ccm-techpreview + steps: + pre: + - chain: ipi-conf-aws + - ref: ingress-aws-conf-lbtype-nlb + - chain: ipi-install + - ref: ccm-conf-apply-feature-gate + - ref: ccm-gather-service-aws + test: + - ref: openshift-e2e-test + post: + - chain: gather-core-dump + - ref: ccm-must-gather + - chain: ipi-aws-post + env: + TEST_SKIPS: >- + Managed cluster should \[apigroup:config.openshift.io\] start all core operators\| + Alerts shouldn't report any unexpected alerts in firing or pending state\| + Prometheus \[apigroup:image.openshift.io\] when installed on the cluster shouldn't report any alerts in firing state apart from Watchdog and AlertmanagerReceiversNotConfigured\| + The HAProxy router should expose prometheus metrics for a route + documentation: |- + The Openshift E2E AWS workflow using CCM as a primary mean to initialize nodes and create external LoadBalancers. + Executes the common end-to-end test suite on AWS to test updated cluster configuration. + Uses TechPreviewNoUpgrade cluster configuration to test CCM functionality, furthermore + it creates NLB for ingress controller to test CCM functionality using Security Groups, enforcing the default behavior + of using Security Groups for NLB.