Merge remote upstream/main into add_validation_dangling_objects

.buildkite/pull-requests.json

@@ -21,7 +21,7 @@
       "pipelineSlug": "package-spec",
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
-      "allowed_list": [ ],
+      "allowed_list": ["dependabot[bot]", "mergify[bot]"],
       "set_commit_status": true,
       "build_on_commit": true,
       "build_on_comment": true,

code/go/internal/yamlschema/loader.go

@@ -11,6 +11,7 @@ import (
 	"path"
 	"sync"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/elastic/gojsonschema"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
@@ -19,6 +20,8 @@ import (
 	"github.com/elastic/package-spec/v2/code/go/internal/spectypes"
 )
 
+var semver3_0_0 = semver.MustParse("3.0.0")
+
 type FileSchemaLoader struct{}
 
 func NewFileSchemaLoader() *FileSchemaLoader {
@@ -42,7 +45,7 @@ type FileSchema struct {
 var formatCheckersMutex sync.Mutex
 
 func (s *FileSchema) Validate(fsys fs.FS, filePath string) ve.ValidationErrors {
-	data, err := loadItemSchema(fsys, filePath, s.options.ContentType)
+	data, err := loadItemSchema(fsys, filePath, s.options.ContentType, s.options.SpecVersion)
 	if err != nil {
 		return ve.ValidationErrors{err}
 	}
@@ -72,24 +75,26 @@ func (s *FileSchema) Validate(fsys fs.FS, filePath string) ve.ValidationErrors {
 	return nil // item content is valid according to the loaded schema
 }
 
-func loadItemSchema(fsys fs.FS, path string, contentType *spectypes.ContentType) ([]byte, error) {
+func loadItemSchema(fsys fs.FS, path string, contentType *spectypes.ContentType, specVersion semver.Version) ([]byte, error) {
 	data, err := fs.ReadFile(fsys, path)
 	if err != nil {
-		return nil, ve.ValidationErrors{errors.Wrap(err, "reading item file failed")}
+		return nil, ve.ValidationErrors{fmt.Errorf("reading item file failed: %w", err)}
 	}
 	if contentType != nil && contentType.MediaType == "application/x-yaml" {
-		return convertYAMLToJSON(data)
+		return convertYAMLToJSON(data, specVersion.LessThan(semver3_0_0))
 	}
 	return data, nil
 }
 
-func convertYAMLToJSON(data []byte) ([]byte, error) {
+func convertYAMLToJSON(data []byte, expandKeys bool) ([]byte, error) {
 	var c interface{}
 	err := yaml.Unmarshal(data, &c)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unmarshalling YAML file failed")
 	}
-	c = expandItemKey(c)
+	if expandKeys {
+		c = expandItemKey(c)
+	}
 
 	data, err = json.Marshal(&c)
 	if err != nil {

code/go/pkg/validator/validator_test.go

@@ -208,6 +208,13 @@ func TestValidateFile(t *testing.T) {
 				"field processors.2.foreach.processor: Additional property paint is not allowed",
 			},
 		},
+		"bad_dotted_fields": {
+			"manifest.yml",
+			[]string{
+				"field conditions: Additional property elastic.subscription is not allowed",
+				"field conditions: Additional property kibana.version is not allowed",
+			},
+		},
 		"bad_dangling_object_ids": {
 			"kibana/dashboard/bad_dangling_object_ids-82273ffe-6acc-4f2f-bbee-c1004abba63d.json",
 			[]string{

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/creasty/defaults v1.7.0
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/gojsonschema v1.2.1
-	github.com/evanphx/json-patch/v5 v5.6.0
+	github.com/evanphx/json-patch/v5 v5.7.0
 	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
 	github.com/otiai10/copy v1.12.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -18,8 +18,8 @@ github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN
 github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
 github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc=
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
-github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
-github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc=
+github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
@@ -29,7 +29,6 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
-github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -46,7 +45,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
 github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
 github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

spec/changelog.yml

@@ -10,6 +10,9 @@
   - description: Stricter validation of index template settings in manifests
     type: breaking-change
     link: https://github.com/elastic/package-spec/pull/587
+  - description: Disallow dotted fields
+    type: breaking-change
+    link: https://github.com/elastic/package-spec/pull/591
   - description: Add validation rule to ensure there are no dangling object IDs
     type: breaking-change
     link: https://github.com/elastic/package-spec/pull/589

test/packages/bad_dangling_object_ids/manifest.yml

@@ -9,8 +9,10 @@ type: integration
 categories:
   - custom
 conditions:
-  kibana.version: "^8.9.1"
-  elastic.subscription: "basic"
+  kibana:
+    version: "^8.9.1"
+  elastic:
+    subscription: "basic"
 screenshots:
   - src: /img/sample-screenshot.png
     title: Sample screenshot

test/packages/bad_dotted_fields/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

test/packages/bad_dotted_fields/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/bad_dotted_fields/docs/README.md

@@ -0,0 +1,84 @@
+<!-- Use this template language as a starting point, replacing {placeholder text} with details about the integration. -->
+<!-- Find more detailed documentation guidelines in https://github.com/elastic/integrations/blob/main/docs/documentation_guidelines.md -->
+
+# Bad Dotted Fields
+
+<!-- The Bad Dotted Fields integration allows you to monitor {name of service}. {name of service} is {describe service}.
+
+Use the Bad Dotted Fields integration to {purpose}. Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference {data stream type} when troubleshooting an issue.
+
+For example, if you wanted to {sample use case} you could {action}. Then you can {visualize|alert|troubleshoot} by {action}. -->
+
+## Data streams
+
+<!-- The Bad Dotted Fields integration collects {one|two} type{s} of data streams: {logs and/or metrics}. -->
+
+<!-- If applicable -->
+<!-- **Logs** help you keep a record of events happening in {service}.
+Log data streams collected by the {name} integration include {sample data stream(s)} and more. See more details in the [Logs](#logs-reference). -->
+
+<!-- If applicable -->
+<!-- **Metrics** give you insight into the state of {service}.
+Metric data streams collected by the {name} integration include {sample data stream(s)} and more. See more details in the [Metrics](#metrics-reference). -->
+
+<!-- Optional: Any additional notes on data streams -->
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+<!--
+	Optional: Other requirements including:
+	* System compatibility
+	* Supported versions of third-party products
+	* Permissions needed
+	* Anything else that could block a user from successfully using the integration
+-->
+
+## Setup
+
+<!-- Any prerequisite instructions -->
+
+For step-by-step instructions on how to set up an integration, see the
+[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide.
+
+<!-- Additional set up instructions -->
+
+<!-- If applicable -->
+<!-- ## Logs reference -->
+
+<!-- Repeat for each data stream of the current type -->
+<!-- ### {Data stream name}
+
+The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
+
+<!-- Optional -->
+<!-- #### Example
+
+An example event for `{data stream name}` looks as following:
+
+{code block with example} -->
+
+<!-- #### Exported fields
+
+{insert table} -->
+
+<!-- If applicable -->
+<!-- ## Metrics reference -->
+
+<!-- Repeat for each data stream of the current type -->
+<!-- ### {Data stream name}
+
+The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
+
+<!-- Optional -->
+<!-- #### Example
+
+An example event for `{data stream name}` looks as following:
+
+{code block with example} -->
+
+<!-- #### Exported fields
+
+{insert table} -->

test/packages/bad_dotted_fields/manifest.yml

@@ -0,0 +1,33 @@
+format_version: 3.0.0
+name: bad_dotted_fields
+title: "Bad Dotted Fields"
+version: 0.0.1
+source:
+  license: "Elastic-2.0"
+description: "This package contains fields with dots."
+type: integration
+categories:
+  - custom
+conditions:
+  kibana.version: "^8.9.1"
+  elastic.subscription: "basic"
+screenshots:
+  - src: /img/sample-screenshot.png
+    title: Sample screenshot
+    size: 600x600
+    type: image/png
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations

test/packages/bad_ingest_pipeline/manifest.yml

@@ -9,8 +9,10 @@ type: integration
 categories:
   - custom
 conditions:
-  kibana.version: "^8.9.1"
-  elastic.subscription: "basic"
+  kibana:
+    version: "^8.9.1"
+  elastic:
+    subscription: "basic"
 screenshots:
   - src: /img/sample-screenshot.png
     title: Sample screenshot

test/packages/good_v3/data_stream/foo/manifest.yml

@@ -44,7 +44,11 @@ streams:
     title: Nginx access logs
     description: Collect Nginx access logs
 dataset_is_prefix: true
-elasticsearch.index_template.mappings:
-  dynamic: strict
-elasticsearch.index_template.ingest_pipeline.name: foobar
-elasticsearch.privileges.indices: [auto_configure, create_doc, monitor]
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: strict
+    ingest_pipeline:
+      name: foobar
+  privileges:
+    indices: [auto_configure, create_doc, monitor]