diff --git a/.circleci/config.yml b/.circleci/config.yml index ba1adacbd5b..d1deb97a5ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,6 +86,26 @@ jobs: jq '. += {"do_nightly_tests": true}' < "<>" > tmpfile mv -v tmpfile "<>" fi + - run: + name: Check for scheduled upgrade run. + command: | + if [[ "scheduled_pipeline" == "<>" + && "upgrade_tests" == "<>" + ]] + then + # add the parameter(s) to the temp file + jq '. += {"do_upgrade_tests": true}' < "<>" > tmpfile + mv -v tmpfile "<>" + fi + - run: + name: Check for labeled upgrade run. + command: | + if gh pr view --json labels --jq ".labels[].name" \ + | grep -w "ci-run-upgrade-tests" + then + jq '. += {"do_upgrade_tests": true}' < "<>" > tmpfile + mv -v tmpfile "<>" + fi - run: name: Set det-version parameter. diff --git a/.circleci/real_config.yml b/.circleci/real_config.yml index 3ba282ebaf7..7c336ef5dd0 100644 --- a/.circleci/real_config.yml +++ b/.circleci/real_config.yml @@ -55,6 +55,9 @@ parameters: do_nightly_tests: type: boolean default: false + do_upgrade_tests: + type: boolean + default: false ee: type: boolean default: false @@ -98,6 +101,10 @@ release-dryrun: &release-dryrun # Tags like: v3.1.3+dryrun - /v\d+\.\d+\.\d+\+dryrun/ +release-upgrade: &release-upgrade + # This value should be updated on every release to capture the updated supported upgrades. + - "release-0.37.0" + upstream-feature-branch: &upstream-feature-branch branches: ignore: @@ -1487,6 +1494,11 @@ commands: command: devcluster --oneshot -c .circleci/devcluster/<> --target-stage <> || circleci-agent step halt background: true + stop-devcluster: + steps: + - run: + command: pkill -SIGHUP -f "devcluster" && rm -rf /tmp/devcluster/lock + make-component: description: "Basic parameterized version of make for building Determined components." parameters: @@ -3170,6 +3182,9 @@ jobs: k8s-version: type: string default: "1.29.5" + upgrade-from-branch: + type: string + default: "" machine: image: <> resource_class: <> @@ -3210,6 +3225,36 @@ jobs: - run: name: Start additionalrm minikube command: minikube start --profile additionalrm --kubernetes-version <> + + - when: + condition: <> + steps: + - run: + name: Get Determined version branch commits. + command: | + git fetch origin <> && git checkout --track origin/<> + - license-gen + - run: make -C master build + - run: make -C agent build + - install-devcluster + - unless: + condition: <> + steps: + - start-devcluster: + devcluster-config: <> + target-stage: <> + - run-e2e-tests: + mark: <> + master-host: localhost + managed-devcluster: <> + extra-pytest-flags: <> + wait-for-master: <> + collect-det-job-logs: <> + - stop-devcluster + - checkout + - run: make -C master build + - run: make -C agent build + - install-devcluster - unless: condition: <> @@ -4721,6 +4766,211 @@ workflows: requires: - package-and-push-system-dev-ee + test-e2e-upgrade: + when: << pipeline.parameters.do_upgrade_tests >> + jobs: + - build-proto + - build-helm + - build-react: + dev-mode: true + - build-docs: + requires: + - build-helm + - build-proto + - build-go: + name: build-go-ee + context: github-read + + - test-e2e: + name: test-e2e-cpu + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 6 + resource-class: xlarge + tf2: true + mark: e2e_cpu + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-cpu-double + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 5 + resource-class: large + tf2: true + mark: e2e_cpu_2a + target-stage: agent2 + devcluster-config: double-priority.devcluster.yaml + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-cpu-oauth + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + devcluster-config: oauth.devcluster.yaml + mark: test_oauth + target-stage: agent1 + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-multi-k8s + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + tf2: true + mark: e2e_multi_k8s + target-stage: master + devcluster-config: multi-k8s.devcluster.yaml + run-minikubes: true + multi-k8s: true + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-single-k8s + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + mark: e2e_single_k8s + target-stage: master + devcluster-config: single-k8s.devcluster.yaml + run-minikubes: true + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-port-registry + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + devcluster-config: port-registry.devcluster.yaml + mark: port_registry + target-stage: agent + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-cpu-elastic + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + resource-class: large + mark: e2e_cpu_elastic + devcluster-config: elastic.devcluster.yaml + target-stage: agent + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-old-agent-versions + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + mark: e2e_cpu_postgres + devcluster-config: custom-agent-version.devcluster.yaml + target-stage: agent + matrix: + parameters: + upgrade-from-branch: *release-upgrade + agent-version: ["0.17.10"] + + - test-e2e: + name: test-e2e-agent-connection-loss + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + mark: e2e_cpu_agent_connection_loss + devcluster-config: agent-no-connection.devcluster.yaml + target-stage: agent + wait-for-master: false + extra-pytest-flags: "--no-compare-stats" + collect-det-job-logs: false + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-saml + context: + - okta + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + devcluster-config: saml.devcluster.yaml + mark: e2e_saml + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-rbac + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + mark: e2e_cpu_rbac + devcluster-config: single-rbac.devcluster.yaml + matrix: + parameters: + upgrade-from-branch: *release-upgrade + + - test-e2e: + name: test-e2e-cpu-model-registry-rbac + context: + - dev-ci-cluster-default-user-credentials + - github-read + requires: + - build-go-ee + parallelism: 1 + devcluster-config: rbac-model-registry.yaml + mark: test_model_registry_rbac + target-stage: agent1 + matrix: + parameters: + upgrade-from-branch: *release-upgrade + test-e2e-longrunning: <<: *do-not-run-on-manual-trigger jobs: