fix: e2e

Signed-off-by: Kirill Kramorov <kirill.kramorov@flant.com>
deckhouse · Oct 1, 2024 · cd2bc40 · cd2bc40
1 parent f60ab8f
commit cd2bc40
Show file tree

Hide file tree

Showing 19 changed files with 2,807 additions and 0 deletions.
diff --git a/.github/ci_templates/e2e_tests.yml b/.github/ci_templates/e2e_tests.yml
@@ -632,6 +632,7 @@ check_e2e_labels:
 {!{-   $runsOnLabel = "e2e-vsphere" -}!}
 {!{- end -}!}
 # <template: e2e_run_job_template>
+{!{ tmpl.Exec "check_e2e_labels_job" . | strings.Indent 2 }!}
 {!{ $ctx.jobID }!}:
   name: "{!{ $ctx.jobName }!}"
   if: ${{ github.event.inputs.cri == '{!{ $ctx.cri }!}' && github.event.inputs.k8s_version == '{!{ $ctx.kubernetesVersion }!}' && github.event.inputs.layout == '{!{ $ctx.layout }!}' }}
@@ -664,6 +665,7 @@ check_e2e_labels:
         DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}}
         DHCTL_PREFIX: ${{ github.event.inputs.cluster_prefix }}
         INSTALL_IMAGE_PATH: ${{ github.event.inputs.installer_image_path }}
+        MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }}
       run: |
         # Create tmppath for test script.
         TMP_DIR_PATH="/mnt/cloud-layouts/layouts/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${DHCTL_PREFIX}"
@@ -693,6 +695,13 @@ check_e2e_labels:
         TERRAFORM_IMAGE_NAME="${BRANCH_REGISTRY_PATH}/e2e-terraform:${DECKHOUSE_IMAGE_TAG}"
 {!{- end }!}
 
+        if [ "${MULTIMASTER}" == true ] ; then
+          MASTERS_COUNT=3
+        else
+          MASTERS_COUNT=1
+        fi
+        echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}"
+
         echo '::echo::on'
         echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT
         echo "install-image-full=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT
@@ -701,6 +710,7 @@ check_e2e_labels:
         echo "terraform-image-name=${TERRAFORM_IMAGE_NAME}" >> $GITHUB_OUTPUT
 {!{- end }!}
         echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT
+        echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT
 
         echo '::echo::off'
 
@@ -729,6 +739,7 @@ check_e2e_labels:
         LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
         TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
         PREFIX: ${{ github.event.inputs.cluster_prefix }}
+        MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }}
         INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-full }}
         DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
 {!{- if eq $provider "eks" }!}

diff --git a/.github/workflows/e2e-abort-aws.yml b/.github/workflows/e2e-abort-aws.yml
diff --git a/.github/workflows/e2e-abort-azure.yml b/.github/workflows/e2e-abort-azure.yml
diff --git a/.github/workflows/e2e-abort-eks.yml b/.github/workflows/e2e-abort-eks.yml
diff --git a/.github/workflows/e2e-abort-gcp.yml b/.github/workflows/e2e-abort-gcp.yml
diff --git a/.github/workflows/e2e-abort-openstack.yml b/.github/workflows/e2e-abort-openstack.yml
diff --git a/.github/workflows/e2e-abort-static.yml b/.github/workflows/e2e-abort-static.yml
diff --git a/.github/workflows/e2e-abort-vcd.yml b/.github/workflows/e2e-abort-vcd.yml
diff --git a/.github/workflows/e2e-abort-vsphere.yml b/.github/workflows/e2e-abort-vsphere.yml
diff --git a/.github/workflows/e2e-abort-yandex-cloud.yml b/.github/workflows/e2e-abort-yandex-cloud.yml
diff --git a/.github/workflows/e2e-aws.yml b/.github/workflows/e2e-aws.yml
@@ -2747,6 +2747,7 @@ jobs:
           REF_FULL: ${{needs.git_info.outputs.ref_full}}
           INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }}
           MANUAL_RUN: "true"
+          MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }}
         run: |
           # Calculate unique prefix for e2e test.
           # GITHUB_RUN_ID is a unique number for each workflow run.
@@ -2828,6 +2829,13 @@ jobs:
           SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]')
           echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}"
 
+          if [ "${MULTIMASTER}" == true ] ; then
+            MASTERS_COUNT=3
+          else
+            MASTERS_COUNT=1
+          fi
+          echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}"
+
           # Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
           echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'."
           docker pull "${INSTALL_IMAGE_NAME}"
@@ -2842,6 +2850,7 @@ jobs:
           echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT
 
           echo '::echo::off'
 
@@ -2857,6 +2866,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
           INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }}
@@ -2879,6 +2889,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -2910,6 +2921,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \
@@ -2964,6 +2976,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
         # <template: e2e_run_template>
@@ -2985,6 +2998,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -2998,6 +3012,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \

diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml
@@ -2787,6 +2787,7 @@ jobs:
           REF_FULL: ${{needs.git_info.outputs.ref_full}}
           INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }}
           MANUAL_RUN: "true"
+          MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }}
         run: |
           # Calculate unique prefix for e2e test.
           # GITHUB_RUN_ID is a unique number for each workflow run.
@@ -2868,6 +2869,13 @@ jobs:
           SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]')
           echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}"
 
+          if [ "${MULTIMASTER}" == true ] ; then
+            MASTERS_COUNT=3
+          else
+            MASTERS_COUNT=1
+          fi
+          echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}"
+
           # Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
           echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'."
           docker pull "${INSTALL_IMAGE_NAME}"
@@ -2882,6 +2890,7 @@ jobs:
           echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT
 
           echo '::echo::off'
 
@@ -2897,6 +2906,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
           INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }}
@@ -2921,6 +2931,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -2952,6 +2963,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \
@@ -3008,6 +3020,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
         # <template: e2e_run_template>
@@ -3031,6 +3044,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -3044,6 +3058,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \

diff --git a/.github/workflows/e2e-eks.yml b/.github/workflows/e2e-eks.yml
@@ -2952,6 +2952,7 @@ jobs:
           REF_FULL: ${{needs.git_info.outputs.ref_full}}
           INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }}
           MANUAL_RUN: "true"
+          MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }}
         run: |
           # Calculate unique prefix for e2e test.
           # GITHUB_RUN_ID is a unique number for each workflow run.
@@ -3037,6 +3038,13 @@ jobs:
           SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]')
           echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}"
 
+          if [ "${MULTIMASTER}" == true ] ; then
+            MASTERS_COUNT=3
+          else
+            MASTERS_COUNT=1
+          fi
+          echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}"
+
           # Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
           echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'."
           docker pull "${INSTALL_IMAGE_NAME}"
@@ -3053,6 +3061,7 @@ jobs:
           echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT
 
           echo '::echo::off'
 
@@ -3068,6 +3077,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           TERRAFORM_IMAGE_NAME: ${{ steps.setup.outputs.terraform-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
@@ -3092,6 +3102,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -3213,6 +3224,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           TERRAFORM_IMAGE_NAME: ${{ steps.setup.outputs.terraform-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
@@ -3236,6 +3248,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing

diff --git a/.github/workflows/e2e-gcp.yml b/.github/workflows/e2e-gcp.yml
@@ -2727,6 +2727,7 @@ jobs:
           REF_FULL: ${{needs.git_info.outputs.ref_full}}
           INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }}
           MANUAL_RUN: "true"
+          MULTIMASTER: ${{ needs.check_e2e_labels.outputs.multimaster }}
         run: |
           # Calculate unique prefix for e2e test.
           # GITHUB_RUN_ID is a unique number for each workflow run.
@@ -2808,6 +2809,13 @@ jobs:
           SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]')
           echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}"
 
+          if [ "${MULTIMASTER}" == true ] ; then
+            MASTERS_COUNT=3
+          else
+            MASTERS_COUNT=1
+          fi
+          echo "Multimaster set ${MULTIMASTER}, MASTERS_COUNT set ${MASTERS_COUNT}"
+
           # Print image name in uppercase to prevent hiding non-secret registry host stored in secret.
           echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'."
           docker pull "${INSTALL_IMAGE_NAME}"
@@ -2822,6 +2830,7 @@ jobs:
           echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT
           echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "masters-count=${MASTERS_COUNT}" >> $GITHUB_OUTPUT
 
           echo '::echo::off'
 
@@ -2837,6 +2846,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count}}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
           INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }}
@@ -2858,6 +2868,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -2889,6 +2900,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \
@@ -2942,6 +2954,7 @@ jobs:
           LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}}
           TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}}
           PREFIX: ${{ steps.setup.outputs.dhctl-prefix}}
+          MASTERS_COUNT: ${{ steps.setup.outputs.masters-count }}
           INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }}
           DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }}
         # <template: e2e_run_template>
@@ -2962,6 +2975,7 @@ jobs:
             LAYOUT=${LAYOUT}
             KUBERNETES_VERSION=${KUBERNETES_VERSION}
             TMP_DIR_PATH=${TMP_DIR_PATH}
+            MASTERS_COUNT=${MASTERS_COUNT}
           "
 
           ls -lh $(pwd)/testing
@@ -2975,6 +2989,7 @@ jobs:
           docker run --rm \
             -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \
             -e PREFIX=${PREFIX} \
+            -e MASTERS_COUNT=${MASTERS_COUNT} \
             -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \
             -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \
             -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \