From 7e4a7a215748609d51557f8f31f095a762a2d283 Mon Sep 17 00:00:00 2001 From: Artem Chernyshev Date: Wed, 30 Oct 2024 15:18:20 +0300 Subject: [PATCH] test: disable secure boot in some of e2e tests Enabled secureboot in Talemu. Split e2e-scaling tests: extract forced removal flows from it. Changed the tests flags to support more complicated machine provision flows: now it can read the config from the yaml file. Signed-off-by: Artem Chernyshev --- .github/workflows/ci.yaml | 85 ++++++++++++++++++- .github/workflows/e2e-backups-cron.yaml | 3 +- .../workflows/e2e-forced-removal-cron.yaml | 79 +++++++++++++++++ .github/workflows/e2e-scaling-cron.yaml | 4 +- .github/workflows/e2e-templates-cron.yaml | 3 +- .github/workflows/slack-notify.yaml | 3 +- .golangci.yml | 4 +- .kres.yaml | 18 +++- cmd/integration-test/pkg/root.go | 49 +++++++++-- cmd/integration-test/pkg/tests/infra.go | 20 ++--- cmd/integration-test/pkg/tests/tests.go | 82 ++++++++++++------ hack/test/integration.sh | 54 +++++++----- hack/test/provisionconfig.yaml | 10 +++ 13 files changed, 339 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/e2e-forced-removal-cron.yaml create mode 100644 hack/test/provisionconfig.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 026937ae..7a3063b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-10-21T13:53:43Z by kres 34e72ac. +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. name: default concurrency: @@ -294,6 +294,7 @@ jobs: xargs -a _out/executable-artifacts -I {} chmod +x {} - name: run-integration-test env: + ENABLE_SECUREBOOT: "false" INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: --test.run CleanState/|EtcdBackupAndRestore RUN_TALEMU_TESTS: "false" @@ -311,6 +312,85 @@ jobs: !~/.talos/clusters/**/swtpm.log retention-days: "5" continue-on-error: true + e2e-forced-removal: + runs-on: + - self-hosted + - omni + if: contains(fromJSON(needs.default.outputs.labels), 'integration/e2e') || contains(fromJSON(needs.default.outputs.labels), 'integration/e2e-forced-removal') + needs: + - default + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.0 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: Mask secrets + run: | + echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')" + - name: Set secrets for job + run: | + sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV" + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: artifacts + path: _out + - name: Fix artifact permissions + run: | + xargs -a _out/executable-artifacts -I {} chmod +x {} + - name: run-integration-test + env: + ENABLE_SECUREBOOT: "false" + INTEGRATION_RUN_E2E_TEST: "false" + INTEGRATION_TEST_ARGS: --test.run CleanState/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/ + RUN_TALEMU_TESTS: "false" + TALEMU_TEST_ARGS: --test.run ImmediateClusterDestruction/|EncryptedCluster/|SinglenodeCluster/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|TalosUpgrades/|KubernetesUpgrades/|MaintenanceDowngrade/|ClusterTemplate/|ScaleUpAndDownAutoProvisionMachineSets/ + WITH_DEBUG: "true" + run: | + sudo -E make run-integration-test + - name: save-talos-logs-artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: talos-logs-e2e-forced-removal + path: |- + ~/.talos/clusters/**/*.log + !~/.talos/clusters/**/swtpm.log + retention-days: "5" + continue-on-error: true e2e-scaling: runs-on: - self-hosted @@ -373,7 +453,7 @@ jobs: - name: run-integration-test env: INTEGRATION_RUN_E2E_TEST: "false" - INTEGRATION_TEST_ARGS: --test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/ + INTEGRATION_TEST_ARGS: --test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism RUN_TALEMU_TESTS: "false" TALEMU_TEST_ARGS: --test.run ImmediateClusterDestruction/|EncryptedCluster/|SinglenodeCluster/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|TalosUpgrades/|KubernetesUpgrades/|MaintenanceDowngrade/|ClusterTemplate/|ScaleUpAndDownAutoProvisionMachineSets/ WITH_DEBUG: "true" @@ -528,6 +608,7 @@ jobs: xargs -a _out/executable-artifacts -I {} chmod +x {} - name: run-integration-test env: + ENABLE_SECUREBOOT: "false" INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: --test.run CleanState/|ClusterTemplate/ RUN_TALEMU_TESTS: "false" diff --git a/.github/workflows/e2e-backups-cron.yaml b/.github/workflows/e2e-backups-cron.yaml index 55d86b1e..3cd02263 100644 --- a/.github/workflows/e2e-backups-cron.yaml +++ b/.github/workflows/e2e-backups-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-09-30T14:56:12Z by kres 8be5fa7. +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. name: e2e-backups-cron concurrency: @@ -60,6 +60,7 @@ jobs: sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV" - name: run-integration-test env: + ENABLE_SECUREBOOT: "false" INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: --test.run CleanState/|EtcdBackupAndRestore RUN_TALEMU_TESTS: "false" diff --git a/.github/workflows/e2e-forced-removal-cron.yaml b/.github/workflows/e2e-forced-removal-cron.yaml new file mode 100644 index 00000000..2bbe6d54 --- /dev/null +++ b/.github/workflows/e2e-forced-removal-cron.yaml @@ -0,0 +1,79 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. + +name: e2e-forced-removal-cron +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true +"on": + schedule: + - cron: 30 1 * * * +jobs: + default: + runs-on: + - self-hosted + - omni + steps: + - name: gather-system-info + id: system-info + uses: kenchan0130/actions-system-info@v1.3.0 + continue-on-error: true + - name: print-system-info + run: | + MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024)) + + OUTPUTS=( + "CPU Core: ${{ steps.system-info.outputs.cpu-core }}" + "CPU Model: ${{ steps.system-info.outputs.cpu-model }}" + "Hostname: ${{ steps.system-info.outputs.hostname }}" + "NodeName: ${NODE_NAME}" + "Kernel release: ${{ steps.system-info.outputs.kernel-release }}" + "Kernel version: ${{ steps.system-info.outputs.kernel-version }}" + "Name: ${{ steps.system-info.outputs.name }}" + "Platform: ${{ steps.system-info.outputs.platform }}" + "Release: ${{ steps.system-info.outputs.release }}" + "Total memory: ${MEMORY_GB} GB" + ) + + for OUTPUT in "${OUTPUTS[@]}";do + echo "${OUTPUT}" + done + continue-on-error: true + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + id: setup-buildx + uses: docker/setup-buildx-action@v3 + with: + driver: remote + endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234 + timeout-minutes: 10 + - name: Mask secrets + run: | + echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')" + - name: Set secrets for job + run: | + sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV" + - name: run-integration-test + env: + ENABLE_SECUREBOOT: "false" + INTEGRATION_RUN_E2E_TEST: "false" + INTEGRATION_TEST_ARGS: --test.run CleanState/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/ + RUN_TALEMU_TESTS: "false" + TALEMU_TEST_ARGS: --test.run ImmediateClusterDestruction/|EncryptedCluster/|SinglenodeCluster/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|TalosUpgrades/|KubernetesUpgrades/|MaintenanceDowngrade/|ClusterTemplate/|ScaleUpAndDownAutoProvisionMachineSets/ + WITH_DEBUG: "true" + run: | + sudo -E make run-integration-test + - name: save-talos-logs-artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: talos-logs + path: |- + ~/.talos/clusters/**/*.log + !~/.talos/clusters/**/swtpm.log + retention-days: "5" diff --git a/.github/workflows/e2e-scaling-cron.yaml b/.github/workflows/e2e-scaling-cron.yaml index d8279e85..4a9d2d99 100644 --- a/.github/workflows/e2e-scaling-cron.yaml +++ b/.github/workflows/e2e-scaling-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-09-30T14:56:12Z by kres 8be5fa7. +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. name: e2e-scaling-cron concurrency: @@ -61,7 +61,7 @@ jobs: - name: run-integration-test env: INTEGRATION_RUN_E2E_TEST: "false" - INTEGRATION_TEST_ARGS: --test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/ + INTEGRATION_TEST_ARGS: --test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism RUN_TALEMU_TESTS: "false" TALEMU_TEST_ARGS: --test.run ImmediateClusterDestruction/|EncryptedCluster/|SinglenodeCluster/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|TalosUpgrades/|KubernetesUpgrades/|MaintenanceDowngrade/|ClusterTemplate/|ScaleUpAndDownAutoProvisionMachineSets/ WITH_DEBUG: "true" diff --git a/.github/workflows/e2e-templates-cron.yaml b/.github/workflows/e2e-templates-cron.yaml index 3a15df1c..f4dcb33d 100644 --- a/.github/workflows/e2e-templates-cron.yaml +++ b/.github/workflows/e2e-templates-cron.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-09-30T14:56:12Z by kres 8be5fa7. +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. name: e2e-templates-cron concurrency: @@ -60,6 +60,7 @@ jobs: sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV" - name: run-integration-test env: + ENABLE_SECUREBOOT: "false" INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: --test.run CleanState/|ClusterTemplate/ RUN_TALEMU_TESTS: "false" diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index fc6bb024..3e50612d 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-08-15T23:44:03Z by kres 7be2a05. +# Generated on 2024-10-30T12:35:59Z by kres 6d3cad4. name: slack-notify "on": @@ -9,6 +9,7 @@ name: slack-notify - default - e2e-short-cron - e2e-scaling-cron + - e2e-forced-removal-cron - e2e-upgrades-cron - e2e-templates-cron - e2e-backups-cron diff --git a/.golangci.yml b/.golangci.yml index 30819c84..79421db8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,14 +1,14 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2024-06-19T21:17:41Z by kres 4c9f215. +# Generated on 2024-10-30T12:55:25Z by kres 6d3cad4. # options for analysis running run: timeout: 10m issues-exit-code: 1 tests: true - modules-download-mode: readonly build-tags: [ ] + modules-download-mode: readonly # output configuration options output: diff --git a/.kres.yaml b/.kres.yaml index 555bf286..b5973f60 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -200,8 +200,21 @@ spec: - integration/e2e-scaling environmentOverride: INTEGRATION_RUN_E2E_TEST: "false" - INTEGRATION_TEST_ARGS: "--test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/" + INTEGRATION_TEST_ARGS: "--test.run CleanState/|ScaleUpAndDown/|ScaleUpAndDownMachineClassBasedMachineSets/|RollingUpdateParallelism" RUN_TALEMU_TESTS: false + - name: e2e-forced-removal + crons: + - '30 1 * * *' + runnerLabels: + - omni + triggerLabels: + - integration/e2e + - integration/e2e-forced-removal + environmentOverride: + INTEGRATION_RUN_E2E_TEST: "false" + INTEGRATION_TEST_ARGS: "--test.run CleanState/|ForcedMachineRemoval/|ReplaceControlPlanes/|ConfigPatching/|KubernetesNodeAudit/" + RUN_TALEMU_TESTS: false + ENABLE_SECUREBOOT: false - name: e2e-upgrades crons: - '30 1 * * *' @@ -226,6 +239,7 @@ spec: INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: "--test.run CleanState/|ClusterTemplate/" RUN_TALEMU_TESTS: false + ENABLE_SECUREBOOT: false - name: e2e-backups crons: - '30 1 * * *' @@ -238,6 +252,7 @@ spec: INTEGRATION_RUN_E2E_TEST: "false" INTEGRATION_TEST_ARGS: "--test.run CleanState/|EtcdBackupAndRestore" RUN_TALEMU_TESTS: false + ENABLE_SECUREBOOT: false - name: e2e-workload-proxy crons: - '30 1 * * *' @@ -581,3 +596,4 @@ spec: - e2e-templates - e2e-upgrades - e2e-backups + - e2e-forced-removal diff --git a/cmd/integration-test/pkg/root.go b/cmd/integration-test/pkg/root.go index f4bb3f51..a5d49427 100644 --- a/cmd/integration-test/pkg/root.go +++ b/cmd/integration-test/pkg/root.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "io" "net/url" "os" "os/exec" @@ -20,6 +21,7 @@ import ( "github.com/mattn/go-shellwords" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" "github.com/siderolabs/omni/client/pkg/compression" clientconsts "github.com/siderolabs/omni/client/pkg/constants" @@ -44,20 +46,51 @@ var rootCmd = &cobra.Command{ testOptions := tests.Options{ RunTestPattern: rootCmdFlags.runTestPattern, - ExpectedMachines: rootCmdFlags.expectedMachines, - CleanupLinks: rootCmdFlags.cleanupLinks, - RunStatsCheck: rootCmdFlags.runStatsCheck, - ProvisionMachines: rootCmdFlags.provisionMachinesCount, + ExpectedMachines: rootCmdFlags.expectedMachines, + CleanupLinks: rootCmdFlags.cleanupLinks, + RunStatsCheck: rootCmdFlags.runStatsCheck, MachineOptions: rootCmdFlags.machineOptions, AnotherTalosVersion: rootCmdFlags.anotherTalosVersion, AnotherKubernetesVersion: rootCmdFlags.anotherKubernetesVersion, OmnictlPath: rootCmdFlags.omnictlPath, - InfraProvider: rootCmdFlags.infraProvider, - ProviderData: rootCmdFlags.providerData, ScalingTimeout: rootCmdFlags.scalingTimeout, } + switch { + case rootCmdFlags.provisionConfigFile != "": + f, err := os.Open(rootCmdFlags.provisionConfigFile) + if err != nil { + return fmt.Errorf("failed to open provision config file %q: %w", rootCmdFlags.provisionConfigFile, err) + } + + decoder := yaml.NewDecoder(f) + + for { + var cfg tests.MachineProvisionConfig + + if err = decoder.Decode(&cfg); err != nil { + if errors.Is(err, io.EOF) { + break + } + + return err + } + + testOptions.ProvisionConfigs = append(testOptions.ProvisionConfigs, cfg) + } + case rootCmdFlags.provisionMachinesCount != 0: + testOptions.ProvisionConfigs = append(testOptions.ProvisionConfigs, + tests.MachineProvisionConfig{ + MachineCount: rootCmdFlags.provisionMachinesCount, + Provider: tests.MachineProviderConfig{ + ID: rootCmdFlags.infraProvider, + Data: rootCmdFlags.providerData, + }, + }, + ) + } + if rootCmdFlags.restartAMachineScript != "" { parsedScript, err := shellwords.Parse(rootCmdFlags.restartAMachineScript) if err != nil { @@ -140,6 +173,7 @@ var rootCmdFlags struct { anotherTalosVersion string anotherKubernetesVersion string omnictlPath string + provisionConfigFile string machineOptions tests.MachineOptions } @@ -175,6 +209,9 @@ var onceInit = sync.OnceValue(func() *cobra.Command { rootCmd.Flags().StringVar(&rootCmdFlags.infraProvider, "infra-provider", "talemu", "use infra provider with the specified ID when provisioning the machines") rootCmd.Flags().StringVar(&rootCmdFlags.providerData, "provider-data", "{}", "the infra provider machine template data to use") rootCmd.Flags().DurationVar(&rootCmdFlags.scalingTimeout, "scale-timeout", time.Second*150, "scale up test timeout") + rootCmd.Flags().StringVar(&rootCmdFlags.provisionConfigFile, "provision-config-file", "", "provision machines with the more complicated configuration") + + rootCmd.MarkFlagsMutuallyExclusive("provision-machines", "provision-config-file") return rootCmd }) diff --git a/cmd/integration-test/pkg/tests/infra.go b/cmd/integration-test/pkg/tests/infra.go index 8474f14d..5307451c 100644 --- a/cmd/integration-test/pkg/tests/infra.go +++ b/cmd/integration-test/pkg/tests/infra.go @@ -30,14 +30,14 @@ import ( // AssertMachinesShouldBeProvisioned creates a machine request set and waits until all requests are fulfilled. // //nolint:gocognit -func AssertMachinesShouldBeProvisioned(testCtx context.Context, client *client.Client, machineCount int, machineRequestSetName, - talosVersion, infraProvider, providerData string, +func AssertMachinesShouldBeProvisioned(testCtx context.Context, client *client.Client, cfg MachineProvisionConfig, machineRequestSetName, + talosVersion string, ) TestFunc { return func(t *testing.T) { ctx, cancel := context.WithTimeout(testCtx, time.Minute*5) defer cancel() - rtestutils.AssertResources(ctx, t, client.Omni().State(), []string{infraProvider}, func(*infra.ProviderStatus, *assert.Assertions) {}) + rtestutils.AssertResources(ctx, t, client.Omni().State(), []string{cfg.Provider.ID}, func(*infra.ProviderStatus, *assert.Assertions) {}) machineRequestSet, err := safe.ReaderGetByID[*omni.MachineRequestSet](ctx, client.Omni().State(), machineRequestSetName) @@ -55,10 +55,10 @@ func AssertMachinesShouldBeProvisioned(testCtx context.Context, client *client.C "siderolabs/" + HelloWorldServiceExtensionName, } - machineRequestSet.TypedSpec().Value.ProviderId = infraProvider + machineRequestSet.TypedSpec().Value.ProviderId = cfg.Provider.ID machineRequestSet.TypedSpec().Value.TalosVersion = talosVersion - machineRequestSet.TypedSpec().Value.ProviderData = providerData - machineRequestSet.TypedSpec().Value.MachineCount = int32(machineCount) + machineRequestSet.TypedSpec().Value.ProviderData = cfg.Provider.Data + machineRequestSet.TypedSpec().Value.MachineCount = int32(cfg.MachineCount) require.NoError(t, client.Omni().State().Create(ctx, machineRequestSet)) @@ -72,8 +72,8 @@ func AssertMachinesShouldBeProvisioned(testCtx context.Context, client *client.C return err } - if resources.Len() != machineCount { - return retry.ExpectedErrorf("provision machine count is %d, expected %d", resources.Len(), machineCount) + if resources.Len() != cfg.MachineCount { + return retry.ExpectedErrorf("provision machine count is %d, expected %d", resources.Len(), cfg.MachineCount) } return nil @@ -89,8 +89,8 @@ func AssertMachinesShouldBeProvisioned(testCtx context.Context, client *client.C return err } - if machines.Len() < machineCount { - return retry.ExpectedErrorf("links count is %d, expected at least %d", machines.Len(), machineCount) + if machines.Len() < cfg.MachineCount { + return retry.ExpectedErrorf("links count is %d, expected at least %d", machines.Len(), cfg.MachineCount) } for r := range resources.All() { diff --git a/cmd/integration-test/pkg/tests/tests.go b/cmd/integration-test/pkg/tests/tests.go index 6d130f3a..fa7c32b8 100644 --- a/cmd/integration-test/pkg/tests/tests.go +++ b/cmd/integration-test/pkg/tests/tests.go @@ -54,14 +54,14 @@ type TalosAPIKeyPrepareFunc func(ctx context.Context, contextName string) error type Options struct { RunTestPattern string - CleanupLinks bool - RunStatsCheck bool - ExpectedMachines int - ProvisionMachines int + CleanupLinks bool + RunStatsCheck bool + ExpectedMachines int RestartAMachineFunc RestartAMachineFunc WipeAMachineFunc WipeAMachineFunc FreezeAMachineFunc FreezeAMachineFunc + ProvisionConfigs []MachineProvisionConfig MachineOptions MachineOptions @@ -69,14 +69,40 @@ type Options struct { AnotherTalosVersion string AnotherKubernetesVersion string OmnictlPath string - InfraProvider string - ProviderData string ScalingTimeout time.Duration } +func (o Options) defaultInfraProvider() string { + if len(o.ProvisionConfigs) == 0 { + return "" + } + + return o.ProvisionConfigs[0].Provider.ID +} + +func (o Options) defaultProviderData() string { + if len(o.ProvisionConfigs) == 0 { + return "{}" + } + + return o.ProvisionConfigs[0].Provider.Data +} + +// MachineProvisionConfig tells the test to provision machines from the infra provider. +type MachineProvisionConfig struct { + Provider MachineProviderConfig `yaml:"provider"` + MachineCount int `yaml:"count"` +} + +// MachineProviderConfig keeps the configuration of the infra provider for the machine provision config. +type MachineProviderConfig struct { + ID string `yaml:"id"` + Data string `yaml:"data"` +} + // Run the integration tests. // -//nolint:maintidx,gocyclo,cyclop +//nolint:maintidx,gocyclo,cyclop,gocognit func Run(ctx context.Context, clientConfig *clientconfig.ClientConfig, options Options) error { rootClient, err := clientConfig.GetClient(ctx) if err != nil { @@ -584,10 +610,10 @@ In between the scaling operations, assert that the cluster is ready and accessib Name: "integration-scaling-auto-provision", ControlPlanes: 1, Workers: 0, - InfraProvider: options.InfraProvider, + InfraProvider: options.defaultInfraProvider(), MachineOptions: options.MachineOptions, - ProviderData: options.ProviderData, + ProviderData: options.defaultProviderData(), ScalingTimeout: options.ScalingTimeout, }), }, @@ -606,9 +632,9 @@ In between the scaling operations, assert that the cluster is ready and accessib Name: "integration-scaling-auto-provision", ControlPlanes: 0, Workers: 1, - InfraProvider: options.InfraProvider, + InfraProvider: options.defaultInfraProvider(), MachineOptions: options.MachineOptions, - ProviderData: options.ProviderData, + ProviderData: options.defaultProviderData(), ScalingTimeout: options.ScalingTimeout, }), }, @@ -628,7 +654,7 @@ In between the scaling operations, assert that the cluster is ready and accessib ControlPlanes: 2, Workers: 0, MachineOptions: options.MachineOptions, - ProviderData: options.ProviderData, + ProviderData: options.defaultInfraProvider(), ScalingTimeout: options.ScalingTimeout, }), }, @@ -647,9 +673,9 @@ In between the scaling operations, assert that the cluster is ready and accessib Name: "integration-scaling-auto-provision", ControlPlanes: 0, Workers: -1, - InfraProvider: options.InfraProvider, + InfraProvider: options.defaultInfraProvider(), MachineOptions: options.MachineOptions, - ProviderData: options.ProviderData, + ProviderData: options.defaultProviderData(), ScalingTimeout: options.ScalingTimeout, }), }, @@ -668,9 +694,9 @@ In between the scaling operations, assert that the cluster is ready and accessib Name: "integration-scaling-auto-provision", ControlPlanes: -2, Workers: 0, - InfraProvider: options.InfraProvider, + InfraProvider: options.defaultInfraProvider(), MachineOptions: options.MachineOptions, - ProviderData: options.ProviderData, + ProviderData: options.defaultProviderData(), }), }, ).Append( @@ -1336,11 +1362,13 @@ Test flow of cluster creation and scaling using cluster templates.`, preRunTests := []testing.InternalTest{} - if options.ProvisionMachines != 0 { - preRunTests = append(preRunTests, testing.InternalTest{ - Name: "AssertMachinesShouldBeProvisioned", - F: AssertMachinesShouldBeProvisioned(ctx, rootClient, options.ProvisionMachines, "main", options.MachineOptions.TalosVersion, options.InfraProvider, options.ProviderData), - }) + if len(options.ProvisionConfigs) != 0 { + for i, cfg := range options.ProvisionConfigs { + preRunTests = append(preRunTests, testing.InternalTest{ + Name: "AssertMachinesShouldBeProvisioned", + F: AssertMachinesShouldBeProvisioned(ctx, rootClient, cfg, fmt.Sprintf("provisioned%d", i), options.MachineOptions.TalosVersion), + }) + } } if len(preRunTests) > 0 { @@ -1357,11 +1385,13 @@ Test flow of cluster creation and scaling using cluster templates.`, postRunTests := []testing.InternalTest{} - if options.ProvisionMachines != 0 { - postRunTests = append(postRunTests, testing.InternalTest{ - Name: "AssertMachinesShouldBeDeprovisioned", - F: AssertMachinesShouldBeDeprovisioned(ctx, rootClient, "main"), - }) + if len(options.ProvisionConfigs) != 0 { + for i := range options.ProvisionConfigs { + postRunTests = append(postRunTests, testing.InternalTest{ + Name: "AssertMachinesShouldBeDeprovisioned", + F: AssertMachinesShouldBeDeprovisioned(ctx, rootClient, fmt.Sprintf("provisioned%d", i)), + }) + } } if options.RunStatsCheck { diff --git a/hack/test/integration.sh b/hack/test/integration.sh index d53b2540..dbd7bbc7 100755 --- a/hack/test/integration.sh +++ b/hack/test/integration.sh @@ -24,6 +24,8 @@ TALOS_VERSION=1.7.6 ARTIFACTS=_out JOIN_TOKEN=testonly RUN_DIR=$(pwd) +ENABLE_SECUREBOOT=${ENABLE_SECUREBOOT:-true} +KERNEL_ARGS_WORKERS_COUNT=2 # Download required artifacts. @@ -41,6 +43,10 @@ SCHEMATIC_ID="cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f" # Build registry mirror args. +if [[ "${ENABLE_SECUREBOOT}" == "false" ]]; then + KERNEL_ARGS_WORKERS_COUNT=4 +fi + if [[ "${CI:-false}" == "true" ]]; then REGISTRY_MIRROR_FLAGS=() @@ -137,7 +143,7 @@ if [[ "${RUN_TALEMU_TESTS:-false}" == "true" ]]; then --talos-version=${TALOS_VERSION} \ --omnictl-path=${ARTIFACTS}/omnictl-linux-amd64 \ --expected-machines=30 \ - --provision-machines=30 \ + --provision-config-file=hack/test/provisionconfig.yaml \ --run-stats-check \ -t 4m \ -p 10 \ @@ -193,7 +199,7 @@ ${ARTIFACTS}/talosctl cluster create \ ${ARTIFACTS}/talosctl cluster create \ --provisioner=qemu \ --controlplanes=1 \ - --workers=2 \ + --workers=${KERNEL_ARGS_WORKERS_COUNT} \ --wait=false \ --mtu=1430 \ --memory=3072 \ @@ -226,27 +232,29 @@ EOF SECURE_BOOT_SCHEMATIC_ID=$(curl -X POST --data-binary "${SECURE_BOOT_SCHEMATIC}" https://factory.talos.dev/schematics | jq -r '.id') -# Kernel args, secure boot -${ARTIFACTS}/talosctl cluster create \ - --provisioner=qemu \ - --controlplanes=1 \ - --workers=1 \ - --wait=false \ - --mtu=1430 \ - --memory=3072 \ - --memory-workers=3072 \ - --cpus=3 \ - --cpus-workers=3 \ - --with-uuid-hostnames \ - \ - --name test-3 \ - --skip-injecting-config \ - --with-init-node \ - --cidr=172.22.0.0/24 \ - --no-masquerade-cidrs=172.20.0.0/24,172.21.0.0/24 \ - --with-tpm2 \ - --iso-path="https://factory.talos.dev/image/${SECURE_BOOT_SCHEMATIC_ID}/v${TALOS_VERSION}/metal-amd64-secureboot.iso" \ - --disk-encryption-key-types=tpm +if [[ "${ENABLE_SECUREBOOT}" == "true" ]]; then + # Kernel args, secure boot + ${ARTIFACTS}/talosctl cluster create \ + --provisioner=qemu \ + --controlplanes=1 \ + --workers=1 \ + --wait=false \ + --mtu=1430 \ + --memory=3072 \ + --memory-workers=3072 \ + --cpus=3 \ + --cpus-workers=3 \ + --with-uuid-hostnames \ + \ + --name test-3 \ + --skip-injecting-config \ + --with-init-node \ + --cidr=172.22.0.0/24 \ + --no-masquerade-cidrs=172.20.0.0/24,172.21.0.0/24 \ + --with-tpm2 \ + --iso-path="https://factory.talos.dev/image/${SECURE_BOOT_SCHEMATIC_ID}/v${TALOS_VERSION}/metal-amd64-secureboot.iso" \ + --disk-encryption-key-types=tpm + fi sleep 5 diff --git a/hack/test/provisionconfig.yaml b/hack/test/provisionconfig.yaml new file mode 100644 index 00000000..88683fad --- /dev/null +++ b/hack/test/provisionconfig.yaml @@ -0,0 +1,10 @@ +--- +count: 15 +provider: + id: talemu +--- +count: 15 +provider: + id: talemu + data: | + secure_boot: true