From ea3d76b35a3717eccfa02376c840198f7a9c66f5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 Jan 2024 17:32:02 +0100 Subject: [PATCH 01/19] Set machineType for testing package steps --- .buildkite/pipeline.trigger.integration.tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 6cc396c90..1a1736696 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -24,6 +24,7 @@ for test in ${STACK_COMMAND_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" + echo " machineType: \"n1-standard-8\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/stack/*/logs/*.log" echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" @@ -42,6 +43,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" + echo " machineType: \"n1-standard-8\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" @@ -62,6 +64,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " UPLOAD_SAFE_LOGS: 1" echo " agents:" echo " provider: \"gcp\"" + echo " machineType: \"n1-standard-8\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package @@ -79,6 +82,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " UPLOAD_SAFE_LOGS: 1" echo " agents:" echo " provider: \"gcp\"" + echo " machineType: \"n1-standard-8\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package From fe867d0296cda0d08c88518ce4a7f9c190e3505b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 Jan 2024 18:35:17 +0100 Subject: [PATCH 02/19] Test steps with Ubuntu 22.04 --- .buildkite/pipeline.trigger.integration.tests.sh | 15 ++++++++++----- .buildkite/pipeline.yml | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 1a1736696..4edd4e06d 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -3,6 +3,9 @@ # exit immediately on failure, or if an undefined variable is used set -eu + +IMAGE_UBUNTU_X86_64=${IMAGE_UBUNTU_X86_64:-"family/core-ubuntu-2004"} + # begin the pipeline.yml file echo "steps:" echo " - group: \":terminal: Integration test suite\"" @@ -24,7 +27,6 @@ for test in ${STACK_COMMAND_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - echo " machineType: \"n1-standard-8\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/stack/*/logs/*.log" echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" @@ -43,7 +45,8 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - echo " machineType: \"n1-standard-8\"" + # echo " machineType: \"n1-standard-8\"" + echo " image: \"${IMAGE_UBUNTU_X86_64}\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" @@ -64,7 +67,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " UPLOAD_SAFE_LOGS: 1" echo " agents:" echo " provider: \"gcp\"" - echo " machineType: \"n1-standard-8\"" + #echo " machineType: \"n1-standard-8\"" + echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package @@ -82,7 +86,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " UPLOAD_SAFE_LOGS: 1" echo " agents:" echo " provider: \"gcp\"" - echo " machineType: \"n1-standard-8\"" + #echo " machineType: \"n1-standard-8\"" + echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package @@ -124,4 +129,4 @@ echo " DEFAULT_VERSION_TAG: v0.80.0" echo " agents:" echo " image: \"${LINUX_AGENT_IMAGE}\"" echo " cpu: \"8\"" -echo " memory: \"4G\"" \ No newline at end of file +echo " memory: \"4G\"" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 888ed0058..c1548e1c7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,6 +41,8 @@ steps: - label: ":pipeline: Trigger Integration tests" command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload" + env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" depends_on: - step: check-static allow_failure: false From 1ec0c2e9ee206108c4918f09a317338409c6b451 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 Jan 2024 19:05:06 +0100 Subject: [PATCH 03/19] Force to use docker compose standalone - Test --- .buildkite/pipeline.trigger.integration.tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 4edd4e06d..a7ba01ad9 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -27,6 +27,8 @@ for test in ${STACK_COMMAND_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" + echo " env:" + echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/stack/*/logs/*.log" echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" @@ -47,6 +49,8 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " provider: \"gcp\"" # echo " machineType: \"n1-standard-8\"" echo " image: \"${IMAGE_UBUNTU_X86_64}\"" + echo " env:" + echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" @@ -65,6 +69,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" + echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" #echo " machineType: \"n1-standard-8\"" @@ -84,6 +89,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" + echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" #echo " machineType: \"n1-standard-8\"" From 35997edd8cebb264614df86083c62c75ee20b697 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 22 Jan 2024 19:43:32 +0100 Subject: [PATCH 04/19] Test docker compose standalone with default OS --- .buildkite/pipeline.trigger.integration.tests.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index a7ba01ad9..3a69f60c9 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -48,7 +48,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " agents:" echo " provider: \"gcp\"" # echo " machineType: \"n1-standard-8\"" - echo " image: \"${IMAGE_UBUNTU_X86_64}\"" + # echo " image: \"${IMAGE_UBUNTU_X86_64}\"" echo " env:" echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" @@ -72,8 +72,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" - #echo " machineType: \"n1-standard-8\"" - echo " image: ${IMAGE_UBUNTU_X86_64}" + # echo " machineType: \"n1-standard-8\"" + # echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package @@ -92,8 +92,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" - #echo " machineType: \"n1-standard-8\"" - echo " image: ${IMAGE_UBUNTU_X86_64}" + # echo " machineType: \"n1-standard-8\"" + # echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package From cc3413d478773036840e011e5ce3643063b13390 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 10:32:04 +0100 Subject: [PATCH 05/19] Install specific docker version --- .buildkite/pipeline.trigger.integration.tests.sh | 8 -------- .buildkite/pipeline.yml | 4 ++-- .buildkite/scripts/install_deps.sh | 15 +++++++++++++++ .buildkite/scripts/integration_tests.sh | 6 ++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 3a69f60c9..b334d84bc 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -4,8 +4,6 @@ set -eu -IMAGE_UBUNTU_X86_64=${IMAGE_UBUNTU_X86_64:-"family/core-ubuntu-2004"} - # begin the pipeline.yml file echo "steps:" echo " - group: \":terminal: Integration test suite\"" @@ -47,8 +45,6 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - # echo " machineType: \"n1-standard-8\"" - # echo " image: \"${IMAGE_UBUNTU_X86_64}\"" echo " env:" echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" @@ -72,8 +68,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" - # echo " machineType: \"n1-standard-8\"" - # echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package @@ -92,8 +86,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" - # echo " machineType: \"n1-standard-8\"" - # echo " image: ${IMAGE_UBUNTU_X86_64}" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/test-coverage/coverage-*.xml" # these files should not be used to compute the final coverage of elastic-package diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c1548e1c7..bb48881e8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,6 +2,8 @@ env: SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" + DOCKER_COMPOSE_VERSION: "v2.18.1" + DOCKER_VERSION: "5:24.0.7-1~ubuntu.20.04~jammy" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0' LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" @@ -41,8 +43,6 @@ steps: - label: ":pipeline: Trigger Integration tests" command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload" - env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" depends_on: - step: check-static allow_failure: false diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 0c10ceacb..4929d2725 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -34,6 +34,21 @@ add_bin_path(){ export PATH="${WORKSPACE}/bin:${PATH}" } +with_docker() { + local ubuntu_version=$(lsb_release -rs) # 20.04 + local ubuntu_codename=$(lsb_release -sc) # focal + sudo apt-get install docker-ce=5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename} +} + +with_docker_compose() { + create_bin_folder + check_platform_architecture + + retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}" + chmod +x ${WORKSPACE}/bin/docker-compose + docker-compose version +} + with_kubernetes() { create_bin_folder check_platform_architecture diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index 101a48a10..c4a778486 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -109,6 +109,12 @@ add_bin_path echo "--- install go" with_go +echo "--- install docker" +with_docker + +echo "--- install docker-compose" +with_docker_compose + if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then echo "--- install kubectl & kind" with_kubernetes From 32f12a6c3d0d578b86d1be72ddd80b0ab5f7d9dc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 10:44:47 +0100 Subject: [PATCH 06/19] Test installing docker-ce/docker-ce-cli --- .buildkite/pipeline.yml | 68 +++++++++++++++--------------- .buildkite/scripts/install_deps.sh | 12 ++++-- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index bb48881e8..93dc0c400 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -10,46 +10,46 @@ env: WINDOWS_AGENT_IMAGE: "family/ci-windows-2022" steps: - - label: ":go: Run check-static" - key: check-static - command: "make check-static" - agents: - image: "${LINUX_AGENT_IMAGE}" - cpu: "8" - memory: "4G" + # - label: ":go: Run check-static" + # key: check-static + # command: "make check-static" + # agents: + # image: "${LINUX_AGENT_IMAGE}" + # cpu: "8" + # memory: "4G" - - label: ":go: :linux: Run unit tests" - key: unit-tests-linux - command: "make test-go-ci" - artifact_paths: - - "build/test-results/*.xml" - - "build/test-coverage/*.xml" - agents: - image: "${LINUX_AGENT_IMAGE}" - cpu: "8" - memory: "4G" + # - label: ":go: :linux: Run unit tests" + # key: unit-tests-linux + # command: "make test-go-ci" + # artifact_paths: + # - "build/test-results/*.xml" + # - "build/test-coverage/*.xml" + # agents: + # image: "${LINUX_AGENT_IMAGE}" + # cpu: "8" + # memory: "4G" - - label: ":go: :windows: Run unit tests" - key: unit-tests-windows - command: ".buildkite/scripts/unit_tests_windows.ps1" - agents: - provider: "gcp" - image: "${WINDOWS_AGENT_IMAGE}" - artifact_paths: - - "TEST-unit.xml" + # - label: ":go: :windows: Run unit tests" + # key: unit-tests-windows + # command: ".buildkite/scripts/unit_tests_windows.ps1" + # agents: + # provider: "gcp" + # image: "${WINDOWS_AGENT_IMAGE}" + # artifact_paths: + # - "TEST-unit.xml" - - wait: ~ - continue_on_failure: true + # - wait: ~ + # continue_on_failure: true - label: ":pipeline: Trigger Integration tests" command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload" - depends_on: - - step: check-static - allow_failure: false - - step: unit-tests-linux - allow_failure: false - - step: unit-tests-windows - allow_failure: false + # depends_on: + # - step: check-static + # allow_failure: false + # - step: unit-tests-linux + # allow_failure: false + # - step: unit-tests-windows + # allow_failure: false - wait: ~ continue_on_failure: true diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 4929d2725..6f4d00423 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -35,9 +35,15 @@ add_bin_path(){ } with_docker() { - local ubuntu_version=$(lsb_release -rs) # 20.04 - local ubuntu_codename=$(lsb_release -sc) # focal - sudo apt-get install docker-ce=5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename} + local ubuntu_version + local ubuntu_codename + ubuntu_version="$(lsb_release -rs)" # 20.04 + ubuntu_codename="$(lsb_release -sc)" # focal + local debian_version="5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" + + sudo apt-get update + sudo apt-get install docker-ce=${debian_version} + sudo apt-get install docker-ce-cli=5:${debian_version} } with_docker_compose() { From 5014a90aa4b42cac69432bdd494b68f3fadc5bda Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 10:54:57 +0100 Subject: [PATCH 07/19] Add sources.list --- .buildkite/scripts/install_deps.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 6f4d00423..2f01f9f1a 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -41,9 +41,12 @@ with_docker() { ubuntu_codename="$(lsb_release -sc)" # focal local debian_version="5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo "deb [arch=$ARCH signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo apt-get install docker-ce=${debian_version} - sudo apt-get install docker-ce-cli=5:${debian_version} + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce=${debian_version}" + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce-cli=${debian_version}" + sudo systemctl start docker } with_docker_compose() { From 4f7b69267ab5077848c9f93b2d71cc14d173ca22 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 10:59:35 +0100 Subject: [PATCH 08/19] Create directory --- .buildkite/scripts/install_deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 2f01f9f1a..f791ee83c 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -41,6 +41,7 @@ with_docker() { ubuntu_codename="$(lsb_release -sc)" # focal local debian_version="5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" + sudo sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo "deb [arch=$ARCH signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update From f35477146f54c1d6f4201188827d56893a5990b2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 11:03:45 +0100 Subject: [PATCH 09/19] Add architecture --- .buildkite/scripts/install_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index f791ee83c..d0d5a3521 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -37,13 +37,15 @@ add_bin_path(){ with_docker() { local ubuntu_version local ubuntu_codename + local architecture ubuntu_version="$(lsb_release -rs)" # 20.04 ubuntu_codename="$(lsb_release -sc)" # focal + architecture=$(dpkg --print-architecture) local debian_version="5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" sudo sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo "deb [arch=$ARCH signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce=${debian_version}" sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce-cli=${debian_version}" From 314570bd03f3d885c0e64afe1fb6e5e32f94e781 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 11:06:37 +0100 Subject: [PATCH 10/19] Allow downgrades --- .buildkite/scripts/install_deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index d0d5a3521..ce57937a8 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -47,8 +47,8 @@ with_docker() { curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce=${debian_version}" - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "docker-ce-cli=${debian_version}" + sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce=${debian_version}" + sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce-cli=${debian_version}" sudo systemctl start docker } From 38d5e5c37596e3c633f8fd93cbec808e0dd5cdfb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 11:12:53 +0100 Subject: [PATCH 11/19] Skip docker installation in container agents --- .buildkite/pipeline.trigger.integration.tests.sh | 7 ++++++- .buildkite/scripts/install_deps.sh | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index b334d84bc..b9b47a679 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -117,13 +117,18 @@ echo " - build/elastic-stack-dump/install-zip-shellinit/logs/*.log" echo " - label: \":go: Running integration test: test-profiles-command\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command" +echo " env:" +echo " DOCKER_COMPOSE_VERSION: \"false\"" echo " agents:" -echo " provider: \"gcp\"" +echo " image: \"${LINUX_AGENT_IMAGE}\"" +echo " cpu: \"8\"" +echo " memory: \"4G\"" echo " - label: \":go: Running integration test: test-check-update-version\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-update-version" echo " env:" echo " DEFAULT_VERSION_TAG: v0.80.0" +echo " DOCKER_COMPOSE_VERSION: \"false\"" echo " agents:" echo " image: \"${LINUX_AGENT_IMAGE}\"" echo " cpu: \"8\"" diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index ce57937a8..45ddb5c6d 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -35,6 +35,10 @@ add_bin_path(){ } with_docker() { + if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then + echo "Skip docker installation" + return + fi local ubuntu_version local ubuntu_codename local architecture @@ -53,6 +57,10 @@ with_docker() { } with_docker_compose() { + if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then + echo "Skip docker installation" + return + fi create_bin_folder check_platform_architecture From 8e1eb37937ae911a80c2fa1a584308f310ef1ed1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 11:55:27 +0100 Subject: [PATCH 12/19] Install docker compose plugin 2.24.1 --- .../pipeline.trigger.integration.tests.sh | 2 + .buildkite/pipeline.yml | 71 +++++++++---------- .buildkite/scripts/install_deps.sh | 21 +++++- .buildkite/scripts/integration_tests.sh | 4 +- 4 files changed, 58 insertions(+), 40 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index b9b47a679..829adc593 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -119,6 +119,7 @@ echo " - label: \":go: Running integration test: test-profiles-command\"" echo " command: ./.buildkite/scripts/integration_tests.sh -t test-profiles-command" echo " env:" echo " DOCKER_COMPOSE_VERSION: \"false\"" +echo " DOCKER_VERSION: \"false\"" echo " agents:" echo " image: \"${LINUX_AGENT_IMAGE}\"" echo " cpu: \"8\"" @@ -129,6 +130,7 @@ echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-u echo " env:" echo " DEFAULT_VERSION_TAG: v0.80.0" echo " DOCKER_COMPOSE_VERSION: \"false\"" +echo " DOCKER_VERSION: \"false\"" echo " agents:" echo " image: \"${LINUX_AGENT_IMAGE}\"" echo " cpu: \"8\"" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93dc0c400..d751de468 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,54 +2,53 @@ env: SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" - DOCKER_COMPOSE_VERSION: "v2.18.1" - DOCKER_VERSION: "5:24.0.7-1~ubuntu.20.04~jammy" + DOCKER_COMPOSE_VERSION: "v2.24.1" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0' LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" WINDOWS_AGENT_IMAGE: "family/ci-windows-2022" steps: - # - label: ":go: Run check-static" - # key: check-static - # command: "make check-static" - # agents: - # image: "${LINUX_AGENT_IMAGE}" - # cpu: "8" - # memory: "4G" + - label: ":go: Run check-static" + key: check-static + command: "make check-static" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" - # - label: ":go: :linux: Run unit tests" - # key: unit-tests-linux - # command: "make test-go-ci" - # artifact_paths: - # - "build/test-results/*.xml" - # - "build/test-coverage/*.xml" - # agents: - # image: "${LINUX_AGENT_IMAGE}" - # cpu: "8" - # memory: "4G" + - label: ":go: :linux: Run unit tests" + key: unit-tests-linux + command: "make test-go-ci" + artifact_paths: + - "build/test-results/*.xml" + - "build/test-coverage/*.xml" + agents: + image: "${LINUX_AGENT_IMAGE}" + cpu: "8" + memory: "4G" - # - label: ":go: :windows: Run unit tests" - # key: unit-tests-windows - # command: ".buildkite/scripts/unit_tests_windows.ps1" - # agents: - # provider: "gcp" - # image: "${WINDOWS_AGENT_IMAGE}" - # artifact_paths: - # - "TEST-unit.xml" + - label: ":go: :windows: Run unit tests" + key: unit-tests-windows + command: ".buildkite/scripts/unit_tests_windows.ps1" + agents: + provider: "gcp" + image: "${WINDOWS_AGENT_IMAGE}" + artifact_paths: + - "TEST-unit.xml" - # - wait: ~ - # continue_on_failure: true + - wait: ~ + continue_on_failure: true - label: ":pipeline: Trigger Integration tests" command: ".buildkite/pipeline.trigger.integration.tests.sh | buildkite-agent pipeline upload" - # depends_on: - # - step: check-static - # allow_failure: false - # - step: unit-tests-linux - # allow_failure: false - # - step: unit-tests-windows - # allow_failure: false + depends_on: + - step: check-static + allow_failure: false + - step: unit-tests-linux + allow_failure: false + - step: unit-tests-windows + allow_failure: false - wait: ~ continue_on_failure: true diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 45ddb5c6d..50e7d96d4 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -35,7 +35,7 @@ add_bin_path(){ } with_docker() { - if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then + if [[ "${DOCKER_VERSION:-"false"}" == "false" ]]; then echo "Skip docker installation" return fi @@ -58,7 +58,7 @@ with_docker() { with_docker_compose() { if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then - echo "Skip docker installation" + echo "Skip docker-compose installation" return fi create_bin_folder @@ -69,6 +69,23 @@ with_docker_compose() { docker-compose version } +with_docker_compose_plugin() { + if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then + echo "Skip docker compose installation (plugin)" + return + fi + create_bin_folder + check_platform_architecture + + k + local DOCKER_CONFIG="$HOME/.docker/cli-plugins" + mkdir -p "$DOCKER_CONFIG" + + retry 5 curl -SL -o ${DOCKER_CONFIG}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}" + chmod +x ${DOCKER_CONFIG}/docker-compose + docker compose version +} + with_kubernetes() { create_bin_folder check_platform_architecture diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index c4a778486..c29fbcfd8 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -112,8 +112,8 @@ with_go echo "--- install docker" with_docker -echo "--- install docker-compose" -with_docker_compose +echo "--- install docker-compose plugin" +with_docker_compose_plugin if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then echo "--- install kubectl & kind" From b09701c248a2474a43c84d4a68e5b6b2f33f0bc4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 12:26:00 +0100 Subject: [PATCH 13/19] Fix typo --- .buildkite/scripts/install_deps.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 50e7d96d4..902e89510 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -77,7 +77,6 @@ with_docker_compose_plugin() { create_bin_folder check_platform_architecture - k local DOCKER_CONFIG="$HOME/.docker/cli-plugins" mkdir -p "$DOCKER_CONFIG" From 7d7fa65df3c1f5c1d64fb7a2a9c0603904f21d99 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 12:37:04 +0100 Subject: [PATCH 14/19] Set docker version --- .buildkite/pipeline.yml | 1 + .buildkite/scripts/install_deps.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d751de468..f91004e87 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,6 +3,7 @@ env: ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_VERSION: "24.0.7-1" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0' LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 902e89510..2d4e043bf 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -45,7 +45,7 @@ with_docker() { ubuntu_version="$(lsb_release -rs)" # 20.04 ubuntu_codename="$(lsb_release -sc)" # focal architecture=$(dpkg --print-architecture) - local debian_version="5:24.0.7-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" + local debian_version="5:${DOCKER_VERSION}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}" sudo sudo mkdir -p /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg From 03046e7007616115701f245b3174fa4936e12b47 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 12:44:44 +0100 Subject: [PATCH 15/19] Remove version suffix --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f91004e87..3a379cc05 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ env: ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" DOCKER_COMPOSE_VERSION: "v2.24.1" - DOCKER_VERSION: "24.0.7-1" + DOCKER_VERSION: "24.0.7" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0' LINUX_AGENT_IMAGE: "golang:${GO_VERSION}" From b87f4be3a0bedae6b7c123865a61bdf380fa1278 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 12:47:09 +0100 Subject: [PATCH 16/19] Remove env variable for standalone --- .buildkite/pipeline.trigger.integration.tests.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index 829adc593..3dd8f559f 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -25,8 +25,6 @@ for test in ${STACK_COMMAND_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - echo " env:" - echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" echo " - build/elastic-stack-dump/stack/*/logs/*.log" echo " - build/elastic-stack-dump/stack/*/logs/fleet-server-internal/**/*" @@ -45,8 +43,6 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " command: ./.buildkite/scripts/integration_tests.sh -t ${test}" echo " agents:" echo " provider: \"gcp\"" - echo " env:" - echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " artifact_paths:" echo " - build/test-results/*.xml" echo " - build/elastic-stack-dump/check-*/logs/*.log" @@ -65,7 +61,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" @@ -83,7 +78,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}" echo " env:" echo " UPLOAD_SAFE_LOGS: 1" - echo " ELASTIC_PACKAGE_COMPOSE_ENABLE_STANDALONE: \"true\"" echo " agents:" echo " provider: \"gcp\"" echo " artifact_paths:" From a563a9169a96d3d04c4592b395618b28efba6d1c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 15:40:35 +0100 Subject: [PATCH 17/19] Remove unused function --- .buildkite/scripts/install_deps.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.buildkite/scripts/install_deps.sh b/.buildkite/scripts/install_deps.sh index 2d4e043bf..3abdb7774 100755 --- a/.buildkite/scripts/install_deps.sh +++ b/.buildkite/scripts/install_deps.sh @@ -56,19 +56,6 @@ with_docker() { sudo systemctl start docker } -with_docker_compose() { - if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then - echo "Skip docker-compose installation" - return - fi - create_bin_folder - check_platform_architecture - - retry 5 curl -SL -o ${WORKSPACE}/bin/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}" - chmod +x ${WORKSPACE}/bin/docker-compose - docker-compose version -} - with_docker_compose_plugin() { if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then echo "Skip docker compose installation (plugin)" From a8e0f1c1ab4dd2f9b4504ce870bc12de013cbb14 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 15:40:42 +0100 Subject: [PATCH 18/19] Test with 2.19.1 version --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3a379cc05..98e83fb2b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ env: SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" - DOCKER_COMPOSE_VERSION: "v2.24.1" + DOCKER_COMPOSE_VERSION: "v2.19.1" DOCKER_VERSION: "24.0.7" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0' From 183eb39495bef4bc0efc773882c3de98c7e7ee97 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 23 Jan 2024 16:00:06 +0100 Subject: [PATCH 19/19] Set docker compose 2.24.1 --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 98e83fb2b..3a379cc05 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ env: SETUP_GVM_VERSION: 'v0.5.1' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151 ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true" ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true" - DOCKER_COMPOSE_VERSION: "v2.19.1" + DOCKER_COMPOSE_VERSION: "v2.24.1" DOCKER_VERSION: "24.0.7" KIND_VERSION: 'v0.20.0' K8S_VERSION: 'v1.29.0'