Merge branch 'main' into handle-multiline-processor

.buildkite/pipeline.trigger.integration.tests.sh

@@ -20,6 +20,7 @@ STACK_COMMAND_TESTS=(
     test-stack-command-7x
     test-stack-command-86
     test-stack-command-8x
+    test-stack-command-9x
     test-stack-command-with-apm-server
 )
 

.buildkite/pipeline.yml

@@ -74,19 +74,21 @@ steps:
       - step: unit-tests-linux
         allow_failure: false
 
-  - label: ":windows: Run stack tests"
-    key: stack-tests-windows
-    command: ".buildkite/scripts/stack_tests_windows.ps1"
-    agents:
-      provider: "gcp"
-      image: "${WINDOWS_AGENT_IMAGE}"
-    depends_on:
-      - step: check-static
-        allow_failure: false
-      - step: unit-tests-windows
-        allow_failure: false
-    artifact_paths:
-      - 'C:\ProgramData\chocolatey\logs\chocolatey.log'
+  # Disable step due to docker errors: https://buildkite.com/elastic/elastic-package/builds/4023#0191ff3a-f18b-4e43-9219-4957da065c37/1827-1829
+  # Related issue: https://github.com/elastic/elastic-package/issues/2107 
+  # - label: ":windows: Run stack tests"
+  #   key: stack-tests-windows
+  #   command: ".buildkite/scripts/stack_tests_windows.ps1"
+  #   agents:
+  #     provider: "gcp"
+  #     image: "${WINDOWS_AGENT_IMAGE}"
+  #   depends_on:
+  #     - step: check-static
+  #       allow_failure: false
+  #     - step: unit-tests-windows
+  #       allow_failure: false
+  #   artifact_paths:
+  #     - 'C:\ProgramData\chocolatey\logs\chocolatey.log'
 
   - wait: ~
     continue_on_failure: true

.buildkite/scripts/stack_tests_windows.ps1

@@ -45,7 +45,7 @@ fixCRLF
 
 withGolang $env:GO_VERSION
 withDocker $env:DOCKER_VERSION
-withDockerCompose $env:DOCKER_COMPOSE_VERSION
+withDockerCompose $env:DOCKER_COMPOSE_VERSION.Substring(1)
 
 Write-Host "--- Docker Info"
 docker info

.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml → .github/workflows/updatecli/updatecli.d/bump-latest-8x-snapshot-version.yml

@@ -1,6 +1,6 @@
 ---
-name: Bump latest elastic-stack test version
-pipelineid: 'bump-latest-elastic-stack-version'
+name: Bump latest 8.x-SNAPSHOT elastic-stack test version
+pipelineid: 'bump-latest-8x-elastic-stack-version'
 
 actions:
   default:
@@ -29,7 +29,7 @@ sources:
     name: Get latest snapshot build
     kind: json
     spec:
-      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      file: https://storage.googleapis.com/artifacts-api/snapshots/8.x.json
       key: .build_id
 
 targets:

.github/workflows/updatecli/updatecli.d/bump-latest-9x-snapshot-version.yml

@@ -0,0 +1,44 @@
+---
+name: Bump latest 9.x-SNAPSHOT elastic-stack test version
+pipelineid: 'bump-latest-9x-elastic-stack-version'
+
+actions:
+  default:
+    title: '[updatecli] Update latest snapshot to {{ source "latestSnapshot" }}'
+    kind: github/pullrequest
+    scmid: default
+    spec:
+      labels:
+        - automation
+        - dependency
+
+scms:
+  default:
+    kind: github
+    spec:
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
+      user: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      token: '{{ requiredEnv "GITHUB_TOKEN" }}'
+      commitusingapi: true
+      branch: main
+
+sources:
+  latestSnapshot:
+    name: Get latest snapshot build
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/snapshots/main.json
+      key: .build_id
+
+targets:
+  update-snapshot:
+    name: '[updatecli] Update latest snapshot to {{ source "latestSnapshot" }}'
+    kind: file
+    sourceid: latestSnapshot
+    scmid: default
+    spec:
+      file: Makefile
+      matchpattern: '(./scripts/test-stack-command.sh) 9\.[^\s]+-SNAPSHOT'
+      replacepattern: '$1 {{ source "latestSnapshot" }}-SNAPSHOT'

.go-version

@@ -1 +1 @@
-1.22.2
+1.23.1

Makefile

@@ -66,22 +66,25 @@ test-stack-command-oldest:
 	./scripts/test-stack-command.sh 7.14.2
 
 test-stack-command-7x:
-	./scripts/test-stack-command.sh 7.17.24-SNAPSHOT
+	./scripts/test-stack-command.sh 7.17.25-SNAPSHOT
 
 # Keeping a test for 8.6 because it has an specific configuration file.
 test-stack-command-86:
 	./scripts/test-stack-command.sh 8.6.2
 
 test-stack-command-8x:
-	./scripts/test-stack-command.sh 8.16.0-a744fd5c-SNAPSHOT
+	./scripts/test-stack-command.sh 8.16.0-4b6c1c4e-SNAPSHOT
+
+test-stack-command-9x:
+	./scripts/test-stack-command.sh 9.0.0-SNAPSHOT
 
 test-stack-command-with-apm-server:
 	APM_SERVER_ENABLED=true ./scripts/test-stack-command.sh
 
 test-stack-command-with-self-monitor:
 	SELF_MONITOR_ENABLED=true ./scripts/test-stack-command.sh
 
-test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-with-apm-server
+test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x test-stack-command-9x test-stack-command-with-apm-server
 
 test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash
 

cmd/create_package.go

@@ -49,7 +49,7 @@ func createPackageCommandAction(cmd *cobra.Command, args []string) error {
 			Name: "type",
 			Prompt: &survey.Select{
 				Message: "Package type:",
-				Options: []string{"input", "integration"},
+				Options: []string{"input", "integration", "content"},
 				Default: "integration",
 			},
 			Validate: survey.Required,

cmd/install.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/install"
+	"github.com/elastic/elastic-package/internal/kibana"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/packages/installer"
 	"github.com/elastic/elastic-package/internal/stack"
@@ -34,6 +35,7 @@ func setupInstallCommand() *cobraext.Command {
 	cmd.Flags().StringP(cobraext.ZipPackageFilePathFlagName, cobraext.ZipPackageFilePathFlagShorthand, "", cobraext.ZipPackageFilePathFlagDescription)
 	cmd.Flags().Bool(cobraext.BuildSkipValidationFlagName, false, cobraext.BuildSkipValidationFlagDescription)
 	cmd.Flags().StringP(cobraext.ProfileFlagName, "p", "", fmt.Sprintf(cobraext.ProfileFlagDescription, install.ProfileNameEnvVar))
+	cmd.Flags().Bool(cobraext.TLSSkipVerifyFlagName, false, cobraext.TLSSkipVerifyFlagDescription)
 
 	return cobraext.NewCommand(cmd, cobraext.ContextPackage)
 }
@@ -57,7 +59,13 @@ func installCommandAction(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	kibanaClient, err := stack.NewKibanaClientFromProfile(profile)
+	var opts []kibana.ClientOption
+	tlsSkipVerify, _ := cmd.Flags().GetBool(cobraext.TLSSkipVerifyFlagName)
+	if tlsSkipVerify {
+		opts = append(opts, kibana.TLSSkipVerify())
+	}
+
+	kibanaClient, err := stack.NewKibanaClientFromProfile(profile, opts...)
 	if err != nil {
 		return fmt.Errorf("could not create kibana client: %w", err)
 	}

go.mod

@@ -2,7 +2,7 @@ module github.com/elastic/elastic-package
 
 go 1.22.1
 
-toolchain go1.22.5
+toolchain go1.23.1
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
@@ -37,46 +37,46 @@ require (
 	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
-	golang.org/x/tools v0.24.0
+	golang.org/x/tools v0.25.0
 	gopkg.in/dnaeon/go-vcr.v3 v3.2.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/gotestsum v1.12.0
-	helm.sh/helm/v3 v3.15.4
+	helm.sh/helm/v3 v3.16.1
 	honnef.co/go/tools v0.5.1
-	k8s.io/apimachinery v0.31.0
-	k8s.io/cli-runtime v0.31.0
-	k8s.io/client-go v0.31.0
+	k8s.io/apimachinery v0.31.1
+	k8s.io/cli-runtime v0.31.1
+	k8s.io/client-go v0.31.1
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
+	dario.cat/mergo v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Masterminds/sprig/v3 v3.3.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/PaesslerAG/gval v1.2.1 // indirect
 	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/Pallinder/go-randomdata v1.2.0 // indirect
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/bitfield/gotestdox v0.2.2 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/creasty/defaults v1.8.0 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dnephin/pflag v1.0.7 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/elastic/gojsonschema v1.2.1 // indirect
 	github.com/elastic/kbncontent v0.1.4 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
+	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
@@ -103,7 +103,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -143,10 +143,10 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cast v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
@@ -161,27 +161,27 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.mongodb.org/mongo-driver v1.11.1 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
-	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/crypto v0.27.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
-	golang.org/x/mod v0.20.0 // indirect
-	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/net v0.29.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.23.0 // indirect
-	golang.org/x/term v0.23.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/term v0.24.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/api v0.31.0 // indirect
-	k8s.io/apiextensions-apiserver v0.30.3 // indirect
-	k8s.io/component-base v0.30.3 // indirect
+	k8s.io/api v0.31.1 // indirect
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/component-base v0.31.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/kubectl v0.30.3 // indirect
+	k8s.io/kubectl v0.31.0 // indirect
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.17.2 // indirect