diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8899c14b4..218917709 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -21,7 +21,7 @@ "pipelineSlug": "package-spec", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_list": ["dependabot[bot]", "mergify[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, diff --git a/code/go/internal/yamlschema/loader.go b/code/go/internal/yamlschema/loader.go index a41065d42..6922b7198 100644 --- a/code/go/internal/yamlschema/loader.go +++ b/code/go/internal/yamlschema/loader.go @@ -11,6 +11,7 @@ import ( "path" "sync" + "github.com/Masterminds/semver/v3" "github.com/elastic/gojsonschema" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -19,6 +20,8 @@ import ( "github.com/elastic/package-spec/v2/code/go/internal/spectypes" ) +var semver3_0_0 = semver.MustParse("3.0.0") + type FileSchemaLoader struct{} func NewFileSchemaLoader() *FileSchemaLoader { @@ -42,7 +45,7 @@ type FileSchema struct { var formatCheckersMutex sync.Mutex func (s *FileSchema) Validate(fsys fs.FS, filePath string) ve.ValidationErrors { - data, err := loadItemSchema(fsys, filePath, s.options.ContentType) + data, err := loadItemSchema(fsys, filePath, s.options.ContentType, s.options.SpecVersion) if err != nil { return ve.ValidationErrors{err} } @@ -72,24 +75,26 @@ func (s *FileSchema) Validate(fsys fs.FS, filePath string) ve.ValidationErrors { return nil // item content is valid according to the loaded schema } -func loadItemSchema(fsys fs.FS, path string, contentType *spectypes.ContentType) ([]byte, error) { +func loadItemSchema(fsys fs.FS, path string, contentType *spectypes.ContentType, specVersion semver.Version) ([]byte, error) { data, err := fs.ReadFile(fsys, path) if err != nil { - return nil, ve.ValidationErrors{errors.Wrap(err, "reading item file failed")} + return nil, ve.ValidationErrors{fmt.Errorf("reading item file failed: %w", err)} } if contentType != nil && contentType.MediaType == "application/x-yaml" { - return convertYAMLToJSON(data) + return convertYAMLToJSON(data, specVersion.LessThan(semver3_0_0)) } return data, nil } -func convertYAMLToJSON(data []byte) ([]byte, error) { +func convertYAMLToJSON(data []byte, expandKeys bool) ([]byte, error) { var c interface{} err := yaml.Unmarshal(data, &c) if err != nil { return nil, errors.Wrapf(err, "unmarshalling YAML file failed") } - c = expandItemKey(c) + if expandKeys { + c = expandItemKey(c) + } data, err = json.Marshal(&c) if err != nil { diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 4e23def42..378408c92 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -208,6 +208,13 @@ func TestValidateFile(t *testing.T) { "field processors.2.foreach.processor: Additional property paint is not allowed", }, }, + "bad_dotted_fields": { + "manifest.yml", + []string{ + "field conditions: Additional property elastic.subscription is not allowed", + "field conditions: Additional property kibana.version is not allowed", + }, + }, "bad_dangling_object_ids": { "kibana/dashboard/bad_dangling_object_ids-82273ffe-6acc-4f2f-bbee-c1004abba63d.json", []string{ diff --git a/go.mod b/go.mod index 2d9e15a8c..796cde151 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/creasty/defaults v1.7.0 github.com/elastic/go-licenser v0.4.1 github.com/elastic/gojsonschema v1.2.1 - github.com/evanphx/json-patch/v5 v5.6.0 + github.com/evanphx/json-patch/v5 v5.7.0 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/otiai10/copy v1.12.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 731f53283..7b3a3da5d 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN github.com/elastic/go-licenser v0.4.1/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= +github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -29,7 +29,6 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -46,7 +45,6 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/spec/changelog.yml b/spec/changelog.yml index 2ebcc66a2..126ad9618 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -10,6 +10,9 @@ - description: Stricter validation of index template settings in manifests type: breaking-change link: https://github.com/elastic/package-spec/pull/587 + - description: Disallow dotted fields + type: breaking-change + link: https://github.com/elastic/package-spec/pull/591 - description: Add validation rule to ensure there are no dangling object IDs type: breaking-change link: https://github.com/elastic/package-spec/pull/589 diff --git a/test/packages/bad_dangling_object_ids/manifest.yml b/test/packages/bad_dangling_object_ids/manifest.yml index dea3296c5..bac21b002 100644 --- a/test/packages/bad_dangling_object_ids/manifest.yml +++ b/test/packages/bad_dangling_object_ids/manifest.yml @@ -9,8 +9,10 @@ type: integration categories: - custom conditions: - kibana.version: "^8.9.1" - elastic.subscription: "basic" + kibana: + version: "^8.9.1" + elastic: + subscription: "basic" screenshots: - src: /img/sample-screenshot.png title: Sample screenshot diff --git a/test/packages/bad_dotted_fields/LICENSE.txt b/test/packages/bad_dotted_fields/LICENSE.txt new file mode 100644 index 000000000..809108b85 --- /dev/null +++ b/test/packages/bad_dotted_fields/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/bad_dotted_fields/changelog.yml b/test/packages/bad_dotted_fields/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/bad_dotted_fields/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/bad_dotted_fields/docs/README.md b/test/packages/bad_dotted_fields/docs/README.md new file mode 100644 index 000000000..f0275deed --- /dev/null +++ b/test/packages/bad_dotted_fields/docs/README.md @@ -0,0 +1,84 @@ + + + +# Bad Dotted Fields + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/packages/bad_dotted_fields/img/sample-logo.svg b/test/packages/bad_dotted_fields/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/bad_dotted_fields/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/bad_dotted_fields/img/sample-screenshot.png b/test/packages/bad_dotted_fields/img/sample-screenshot.png new file mode 100644 index 000000000..d7a56a3ec Binary files /dev/null and b/test/packages/bad_dotted_fields/img/sample-screenshot.png differ diff --git a/test/packages/bad_dotted_fields/manifest.yml b/test/packages/bad_dotted_fields/manifest.yml new file mode 100644 index 000000000..b689e750c --- /dev/null +++ b/test/packages/bad_dotted_fields/manifest.yml @@ -0,0 +1,33 @@ +format_version: 3.0.0 +name: bad_dotted_fields +title: "Bad Dotted Fields" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "This package contains fields with dots." +type: integration +categories: + - custom +conditions: + kibana.version: "^8.9.1" + elastic.subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations diff --git a/test/packages/bad_ingest_pipeline/manifest.yml b/test/packages/bad_ingest_pipeline/manifest.yml index 4aa928089..add099bdf 100644 --- a/test/packages/bad_ingest_pipeline/manifest.yml +++ b/test/packages/bad_ingest_pipeline/manifest.yml @@ -9,8 +9,10 @@ type: integration categories: - custom conditions: - kibana.version: "^8.9.1" - elastic.subscription: "basic" + kibana: + version: "^8.9.1" + elastic: + subscription: "basic" screenshots: - src: /img/sample-screenshot.png title: Sample screenshot diff --git a/test/packages/good_v3/data_stream/foo/manifest.yml b/test/packages/good_v3/data_stream/foo/manifest.yml index d4654d7e9..568581bf8 100644 --- a/test/packages/good_v3/data_stream/foo/manifest.yml +++ b/test/packages/good_v3/data_stream/foo/manifest.yml @@ -44,7 +44,11 @@ streams: title: Nginx access logs description: Collect Nginx access logs dataset_is_prefix: true -elasticsearch.index_template.mappings: - dynamic: strict -elasticsearch.index_template.ingest_pipeline.name: foobar -elasticsearch.privileges.indices: [auto_configure, create_doc, monitor] +elasticsearch: + index_template: + mappings: + dynamic: strict + ingest_pipeline: + name: foobar + privileges: + indices: [auto_configure, create_doc, monitor] diff --git a/test/packages/good_v3/data_stream/hidden_data_stream/manifest.yml b/test/packages/good_v3/data_stream/hidden_data_stream/manifest.yml index be7a5ce5e..945285c87 100644 --- a/test/packages/good_v3/data_stream/hidden_data_stream/manifest.yml +++ b/test/packages/good_v3/data_stream/hidden_data_stream/manifest.yml @@ -2,5 +2,6 @@ title: Hidden data stream and ilm policy overrride type: metrics hidden: true elasticsearch: - index_template.mappings: - dynamic: false + index_template: + mappings: + dynamic: false diff --git a/test/packages/good_v3/data_stream/ilm_policy/manifest.yml b/test/packages/good_v3/data_stream/ilm_policy/manifest.yml index b4f11ea87..985aed3c7 100644 --- a/test/packages/good_v3/data_stream/ilm_policy/manifest.yml +++ b/test/packages/good_v3/data_stream/ilm_policy/manifest.yml @@ -2,5 +2,6 @@ title: Hidden data stream and ilm policy overrride type: metrics ilm_policy: metrics-good_v3.ilm_policy-diagnostics elasticsearch: - index_template.mappings: - dynamic: false + index_template: + mappings: + dynamic: false diff --git a/test/packages/good_v3/elasticsearch/transform/metadata_current/manifest.yml b/test/packages/good_v3/elasticsearch/transform/metadata_current/manifest.yml index 52e5191e1..11a817930 100644 --- a/test/packages/good_v3/elasticsearch/transform/metadata_current/manifest.yml +++ b/test/packages/good_v3/elasticsearch/transform/metadata_current/manifest.yml @@ -2,12 +2,13 @@ destination_index_template: settings: index: codec: best_compression - sort.field: - - "@timestamp" - - agent.id - sort.order: - - desc - - asc + sort: + field: + - "@timestamp" + - agent.id + order: + - desc + - asc mappings: dynamic: true dynamic_templates: diff --git a/test/packages/good_v3/manifest.yml b/test/packages/good_v3/manifest.yml index a92f6fee8..9260fd9ec 100644 --- a/test/packages/good_v3/manifest.yml +++ b/test/packages/good_v3/manifest.yml @@ -7,10 +7,12 @@ type: integration source: license: "Apache-2.0" conditions: - kibana.version: '^8.10.0' - elastic.subscription: 'basic' - elastic.capabilities: - - observability + kibana: + version: '^8.10.0' + elastic: + subscription: 'basic' + capabilities: + - observability vars: - name: package_password type: password @@ -61,4 +63,6 @@ icons: # /main is a specific action underneath the monitor privilege. Declaring # "monitor/main" limits the provided privilege, "monitor", to only the "main" # action. -elasticsearch.privileges.cluster: [monitor/main] +elasticsearch: + privileges: + cluster: [monitor/main]