diff --git a/code/go/internal/loader/spec.go b/code/go/internal/loader/spec.go index 662eaead3..645ee1524 100644 --- a/code/go/internal/loader/spec.go +++ b/code/go/internal/loader/spec.go @@ -5,7 +5,10 @@ package loader import ( + "errors" + "fmt" "io/fs" + "os" "github.com/Masterminds/semver/v3" @@ -18,5 +21,12 @@ import ( func LoadSpec(fsys fs.FS, version semver.Version, pkgType string) (spectypes.ItemSpec, error) { fileSpecLoader := yamlschema.NewFileSchemaLoader() loader := specschema.NewFolderSpecLoader(fsys, fileSpecLoader, version) - return loader.Load(pkgType) + spec, err := loader.Load(pkgType) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("package type %q not supported (%w)", pkgType, err) + } + if err != nil { + return nil, err + } + return spec, nil } diff --git a/code/go/internal/spec_test.go b/code/go/internal/spec_test.go index 599937599..d4783d7f9 100644 --- a/code/go/internal/spec_test.go +++ b/code/go/internal/spec_test.go @@ -19,6 +19,7 @@ func TestLoadAllBundledVersions(t *testing.T) { require.NoError(t, err) for _, version := range versions { + testForVersionType(t, version, "content") testForVersionType(t, version, "input") testForVersionType(t, version, "integration") } diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index 866c8d14c..d4ca35425 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -196,12 +196,12 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.WarnOn(semantic.ValidateMinimumKibanaVersion), until: semver.MustParse("3.0.0")}, {fn: semantic.ValidateMinimumKibanaVersion, since: semver.MustParse("3.0.0")}, {fn: semantic.ValidateFieldGroups}, - {fn: semantic.ValidateFieldsLimits(rootSpec.MaxFieldsPerDataStream())}, - {fn: semantic.ValidateUniqueFields, since: semver.MustParse("2.0.0")}, - {fn: semantic.ValidateDimensionFields}, - {fn: semantic.ValidateDateFields}, - {fn: semantic.ValidateRequiredFields}, - {fn: semantic.ValidateExternalFieldsWithDevFolder}, + {fn: semantic.ValidateFieldsLimits(rootSpec.MaxFieldsPerDataStream()), types: []string{"integration", "input"}}, + {fn: semantic.ValidateUniqueFields, since: semver.MustParse("2.0.0"), types: []string{"integration", "input"}}, + {fn: semantic.ValidateDimensionFields, types: []string{"integration", "input"}}, + {fn: semantic.ValidateDateFields, types: []string{"integration", "input"}}, + {fn: semantic.ValidateRequiredFields, types: []string{"integration", "input"}}, + {fn: semantic.ValidateExternalFieldsWithDevFolder, types: []string{"integration", "input"}}, {fn: semantic.WarnOn(semantic.ValidateVisualizationsUsedByValue), types: []string{"integration"}, until: semver.MustParse("3.0.0")}, {fn: semantic.ValidateVisualizationsUsedByValue, types: []string{"integration"}, since: semver.MustParse("3.0.0")}, {fn: semantic.ValidateILMPolicyPresent, since: semver.MustParse("2.0.0"), types: []string{"integration"}}, diff --git a/code/go/pkg/specerrors/filter.go b/code/go/pkg/specerrors/filter.go index 43a64c540..3ab6cb3ea 100644 --- a/code/go/pkg/specerrors/filter.go +++ b/code/go/pkg/specerrors/filter.go @@ -23,6 +23,8 @@ type Filter struct { type FilterResult struct { Processed error Removed error + + UnusedProcessors []Processor } // Run runs all the processors over all the validation errors and return the filtered ones @@ -30,6 +32,7 @@ func (r *Filter) Run(allErrors ValidationErrors) (FilterResult, error) { newErrors := allErrors var allFiltered ValidationErrors + var unused []Processor for _, p := range r.processors { result, err := p.Process(newErrors) if err != nil { @@ -37,11 +40,16 @@ func (r *Filter) Run(allErrors ValidationErrors) (FilterResult, error) { } newErrors = result.Processed allFiltered.Append(result.Removed) + + if len(result.Removed) == 0 { + unused = append(unused, p) + } } return FilterResult{ - Processed: nilOrValidationErrors(newErrors), - Removed: nilOrValidationErrors(allFiltered), + Processed: nilOrValidationErrors(newErrors), + Removed: nilOrValidationErrors(allFiltered), + UnusedProcessors: unused, }, nil } diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index e346161e1..24b1b635b 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -33,6 +33,7 @@ func TestValidateFile(t *testing.T) { "good_v2": {}, "good_v3": {}, "good_input": {}, + "good_content": {}, "deploy_custom_agent": {}, "deploy_custom_agent_multi_services": {}, "deploy_docker": {}, @@ -232,6 +233,7 @@ func TestValidateFile(t *testing.T) { filter := specerrors.NewFilter(filterConfig) result, err := filter.Run(verrs) require.NoError(t, err) + assert.Empty(t, result.UnusedProcessors, "There are unused exclusion checks in the validation.yml file") errs = result.Processed } } diff --git a/spec/changelog.yml b/spec/changelog.yml index 270286872..a56f79e09 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -2,11 +2,24 @@ ## This file documents changes in the package specification. It is NOT a package specification file. ## Newer entries go at the bottom of each in-development version. ## +- version: 3.4.0-next + changes: + - description: Add support for content packages. + type: enhancement + link: https://github.com/elastic/package-spec/pull/777 - version: 3.3.0-next changes: - description: Add support for `slo` assets. type: enhancement link: https://github.com/elastic/package-spec/pull/767 + - description: Add spec for img directory. + type: enhancement + link: https://github.com/elastic/package-spec/pull/777 +- version: 3.2.2-next + changes: + - description: Improved error message for unsupported package types. + type: enhancement + link: https://github.com/elastic/package-spec/pull/777 - version: 3.2.1 changes: - description: Improve error information for missing dataset field in manifest. diff --git a/spec/content/kibana/spec.yml b/spec/content/kibana/spec.yml new file mode 100644 index 000000000..90ec23d9a --- /dev/null +++ b/spec/content/kibana/spec.yml @@ -0,0 +1,31 @@ +spec: + additionalContents: false + contents: + - description: Folder containing Kibana dashboard assets + type: folder + name: dashboard + required: false + contents: + - description: A dashboard asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: File containing saved object tag definitions for assets + type: file + contentMediaType: "application/x-yaml" + name: "tags.yml" + required: false + $ref: "../../integration/kibana/tags.spec.yml" + - description: Folder containing Kibana SLO assets + type: folder + name: slo + required: false + contents: + - description: An SLO asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden diff --git a/spec/content/manifest.spec.yml b/spec/content/manifest.spec.yml new file mode 100644 index 000000000..40e0334f7 --- /dev/null +++ b/spec/content/manifest.spec.yml @@ -0,0 +1,70 @@ +## +## Describes the specification for the content package's main manifest.yml file +## +spec: + # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability + type: object + additionalProperties: false + definitions: + discovery: + description: > + Description of the data this package can be used with. It can be used to discover + the package from elements in the existing data. + type: object + additionalProperties: false + properties: + fields: + description: Description of the fields this package can be used with. + type: array + items: + type: object + properties: + name: + description: Name of the field. + type: string + properties: + format_version: + description: The version of the package specification format used by this package. + $ref: "../integration/manifest.spec.yml#/definitions/version" + name: + description: The name of the package. + type: string + pattern: '^[a-z0-9_]+$' + examples: + - apache + title: + $ref: "../integration/manifest.spec.yml#/definitions/title" + description: + $ref: "../integration/manifest.spec.yml#/definitions/description" + version: + description: The version of the package. + $ref: "../integration/manifest.spec.yml#/definitions/version" + source: + $ref: "../integration/manifest.spec.yml#/definitions/source" + type: + description: The type of package. + type: string + enum: + - content + examples: + - content + categories: + $ref: "../integration/manifest.spec.yml#/definitions/categories" + conditions: + $ref: "../integration/manifest.spec.yml#/definitions/conditions" + discovery: + $ref: "#/definitions/discovery" + icons: + $ref: "../integration/manifest.spec.yml#/definitions/icons" + screenshots: + $ref: "../integration/manifest.spec.yml#/definitions/screenshots" + owner: + $ref: "../integration/manifest.spec.yml#/definitions/owner" + required: + - format_version + - name + - title + - description + - version + - type + - owner diff --git a/spec/content/spec.yml b/spec/content/spec.yml new file mode 100644 index 000000000..f5b23ef5b --- /dev/null +++ b/spec/content/spec.yml @@ -0,0 +1,60 @@ +## +## Entrypoint of "content packages" specification. +## The specification is considered "beta" at the moment, so it may change until we release it as GA. +## +## Describes the folders and files that make up a package. +## +spec: + additionalContents: false + totalContentsLimit: 65535 + totalSizeLimit: 250MB + sizeLimit: 150MB + configurationSizeLimit: 5MB + relativePathSizeLimit: 3MB + release: beta + contents: + - description: The main package manifest file + type: file + contentMediaType: "application/x-yaml" + sizeLimit: 5MB + name: "manifest.yml" + required: true + $ref: "./manifest.spec.yml" + - description: The package's CHANGELOG file + type: file + contentMediaType: "application/x-yaml" + name: "changelog.yml" + required: true + $ref: "../integration/changelog.spec.yml" + - description: The package's license file + type: file + contentMediaType: "text/plain" + name: "LICENSE.txt" + required: false + - description: Folder containing documentation for the package + type: folder + name: docs + required: true + $ref: "../integration/docs/spec.yml" + - description: Folder containing images for the package + type: folder + name: img + required: false + $ref: "../integration/img/spec.yml" + - description: Folder containing Kibana assets provided by the package + type: folder + name: kibana + required: false + $ref: "./kibana/spec.yml" + - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package. + type: file + contentMediaType: "application/x-yaml" + name: "validation.yml" + required: false + $ref: "../integration/validation.spec.yml" + +versions: + - before: 3.4.0 + patch: + - op: remove + path: "/contents" # Package type not available before this version. diff --git a/spec/input/spec.yml b/spec/input/spec.yml index 282c4672a..bdf69e70a 100644 --- a/spec/input/spec.yml +++ b/spec/input/spec.yml @@ -1,6 +1,5 @@ ## ## Entrypoint of "input packages" specification. -## The specification is considered "beta" at the moment, so it may change until we release it as GA. ## ## Describes the folders and files that make up a package. ## @@ -69,8 +68,17 @@ spec: name: "validation.yml" required: false $ref: "../integration/validation.spec.yml" + - description: Folder containing images for the package + type: folder + name: img + required: false + $ref: "../integration/img/spec.yml" versions: + - before: 3.3.0 + patch: + - op: remove + path: "/contents/10" # Definition for img folder. - before: 2.10.0 patch: - op: remove diff --git a/spec/integration/docs/spec.yml b/spec/integration/docs/spec.yml index b8f673e00..691073ee4 100644 --- a/spec/integration/docs/spec.yml +++ b/spec/integration/docs/spec.yml @@ -1,13 +1,13 @@ - spec: - additionalContents: false - contents: - - description: Main README file - type: file - contentMediaType: "text/markdown" - name: "README.md" - required: true - - description: Other README files (can be used by policy templates) - type: file - contentMediaType: "text/markdown" - pattern: '^.+.md' - required: false +spec: + additionalContents: false + contents: + - description: Main README file + type: file + contentMediaType: "text/markdown" + name: "README.md" + required: true + - description: Other README files (can be used by policy templates) + type: file + contentMediaType: "text/markdown" + pattern: '^.+.md' + required: false diff --git a/spec/integration/img/spec.yml b/spec/integration/img/spec.yml new file mode 100644 index 000000000..7b156eaf8 --- /dev/null +++ b/spec/integration/img/spec.yml @@ -0,0 +1,2 @@ +spec: + additionalContents: true # TODO: Add definition for this dir? diff --git a/spec/integration/kibana/tags.spec.yml b/spec/integration/kibana/tags.spec.yml index 102cc5526..5b446042e 100644 --- a/spec/integration/kibana/tags.spec.yml +++ b/spec/integration/kibana/tags.spec.yml @@ -1,5 +1,5 @@ ## -## Describes the specification for a routing rules yml file +## Describes the specification for the additional kibana tags provided by the package. ## spec: # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability diff --git a/spec/integration/spec.yml b/spec/integration/spec.yml index 574cf1ea3..066e8e89b 100644 --- a/spec/integration/spec.yml +++ b/spec/integration/spec.yml @@ -72,3 +72,14 @@ spec: name: "validation.yml" required: false $ref: "./validation.spec.yml" + - description: Folder containing images for the package + type: folder + name: img + required: false + $ref: "./img/spec.yml" + +versions: + - before: 3.3.0 + patch: + - op: remove + path: "/contents/11" # Definition for img folder. diff --git a/test/packages/good_content/changelog.yml b/test/packages/good_content/changelog.yml new file mode 100644 index 000000000..c8397a8b6 --- /dev/null +++ b/test/packages/good_content/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.1.0 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/777 diff --git a/test/packages/good_content/docs/README.md b/test/packages/good_content/docs/README.md new file mode 100644 index 000000000..3a6090d84 --- /dev/null +++ b/test/packages/good_content/docs/README.md @@ -0,0 +1 @@ +# Reference package of content type diff --git a/test/packages/good_content/img/kibana-system.png b/test/packages/good_content/img/kibana-system.png new file mode 100644 index 000000000..8741a5662 Binary files /dev/null and b/test/packages/good_content/img/kibana-system.png differ diff --git a/test/packages/good_content/img/system.svg b/test/packages/good_content/img/system.svg new file mode 100644 index 000000000..0aba96275 --- /dev/null +++ b/test/packages/good_content/img/system.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/good_content/manifest.yml b/test/packages/good_content/manifest.yml new file mode 100644 index 000000000..be8a26f0a --- /dev/null +++ b/test/packages/good_content/manifest.yml @@ -0,0 +1,32 @@ +format_version: 3.4.0 +name: good_content +title: Good content package +description: > + This package is a dummy example for packages with the content type. + These packages contain resources that are useful with data ingested by other integrations. + They are not used to configure data sources. +version: 0.1.0 +type: content +source: + license: "Apache-2.0" +conditions: + kibana: + version: '^8.16.0' #TBD + elastic: + subscription: 'basic' +discovery: + fields: + - name: process.pid +screenshots: + - src: /img/kibana-system.png + title: kibana system + size: 1220x852 + type: image/png +icons: + - src: /img/system.svg + title: system + size: 1000x1000 + type: image/svg+xml +owner: + github: elastic/ecosystem + type: elastic diff --git a/test/packages/good_content/validation.yml b/test/packages/good_content/validation.yml new file mode 100644 index 000000000..fa39a1d6f --- /dev/null +++ b/test/packages/good_content/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - PSR00002 # Allow to use non-GA features.