Initial definition of content pack[age]s

code/go/internal/loader/spec.go

@@ -5,7 +5,10 @@
 package loader
 
 import (
+	"errors"
+	"fmt"
 	"io/fs"
+	"os"
 
 	"github.com/Masterminds/semver/v3"
 
@@ -18,5 +21,12 @@ import (
 func LoadSpec(fsys fs.FS, version semver.Version, pkgType string) (spectypes.ItemSpec, error) {
 	fileSpecLoader := yamlschema.NewFileSchemaLoader()
 	loader := specschema.NewFolderSpecLoader(fsys, fileSpecLoader, version)
-	return loader.Load(pkgType)
+	spec, err := loader.Load(pkgType)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("package type %q not supported (%w)", pkgType, err)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return spec, nil
 }

code/go/internal/spec_test.go

@@ -19,6 +19,7 @@ func TestLoadAllBundledVersions(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, version := range versions {
+		testForVersionType(t, version, "content")
 		testForVersionType(t, version, "input")
 		testForVersionType(t, version, "integration")
 	}

code/go/pkg/specerrors/filter.go

@@ -23,25 +23,33 @@ type Filter struct {
 type FilterResult struct {
 	Processed error
 	Removed   error
+
+	UnusedProcessors []Processor
 }
 
 // Run runs all the processors over all the validation errors and return the filtered ones
 func (r *Filter) Run(allErrors ValidationErrors) (FilterResult, error) {
 	newErrors := allErrors
 	var allFiltered ValidationErrors
 
+	var unused []Processor
 	for _, p := range r.processors {
 		result, err := p.Process(newErrors)
 		if err != nil {
 			return FilterResult{Processed: allErrors, Removed: nil}, err
 		}
 		newErrors = result.Processed
 		allFiltered.Append(result.Removed)
+
+		if len(result.Removed) == 0 {
+			unused = append(unused, p)
+		}
 	}
 
 	return FilterResult{
-		Processed: nilOrValidationErrors(newErrors),
-		Removed:   nilOrValidationErrors(allFiltered),
+		Processed:        nilOrValidationErrors(newErrors),
+		Removed:          nilOrValidationErrors(allFiltered),
+		UnusedProcessors: unused,
 	}, nil
 }
 

code/go/pkg/validator/validator_test.go

@@ -33,6 +33,7 @@ func TestValidateFile(t *testing.T) {
 		"good_v2":                            {},
 		"good_v3":                            {},
 		"good_input":                         {},
+		"good_content":                       {},
 		"deploy_custom_agent":                {},
 		"deploy_custom_agent_multi_services": {},
 		"deploy_docker":                      {},
@@ -232,6 +233,7 @@ func TestValidateFile(t *testing.T) {
 					filter := specerrors.NewFilter(filterConfig)
 					result, err := filter.Run(verrs)
 					require.NoError(t, err)
+					assert.Empty(t, result.UnusedProcessors, "There are unused exclusion checks in the validation.yml file")
 					errs = result.Processed
 				}
 			}

spec/changelog.yml

@@ -2,11 +2,21 @@
 ## This file documents changes in the package specification. It is NOT a package specification file.
 ## Newer entries go at the bottom of each in-development version.
 ##
+- version: 3.4.0-next
+  changes:
+  - description: Add support for content packages.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.3.0-next
   changes:
   - description: Add support for `slo` assets.
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/767
+- version: 3.2.2-next
+  changes:
+  - description: Improved error message for unsupported package types.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.2.1
   changes:
   - description: Improve error information for missing dataset field in manifest.

spec/content/kibana/spec.yml

@@ -0,0 +1,31 @@
+spec:
+  additionalContents: false
+  contents:
+  - description: Folder containing Kibana dashboard assets
+    type: folder
+    name: dashboard
+    required: false
+    contents:
+    - description: A dashboard asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+  - description: File containing saved object tag definitions for assets
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "tags.yml"
+    required: false
+    $ref: "./tags.spec.yml"
+  - description: Folder containing Kibana SLO assets
+    type: folder
+    name: slo
+    required: false
+    contents:
+    - description: An SLO asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden

spec/content/kibana/tags.spec.yml

@@ -0,0 +1,49 @@
+##
+## Describes the specification for a routing rules yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: array
+  description: Tags automatically added to saved object assets.
+  items:
+     type: object
+     additionalProperties: false
+     properties:
+       text:
+         description: Tag name.
+         type: string
+       asset_types:
+         description: >
+           This tag will be added to all the assets of these types included in the package.
+           If two or more pacakges define the same tag, there will be just one tag created in Kibana and
+           all the assets will be using the same tag.
+         type: array
+         items:
+           type: string
+           enum: # added the same types that are allowed under kibana folder
+             - dashboard
+             - visualization
+             - search
+             - map
+             - lens
+             - index_pattern
+             - security_rule
+             - csp_rule_template
+             - ml_module
+             - osquery_pack_asset
+             - osquery_saved_query
+       asset_ids:
+         description: >
+           Asset IDs where this tag is going to be added.
+           If two or more pacakges define the same tag, there will be just one tag created in Kibana and
+           all the assets will be using the same tag.
+         type: array
+         items:
+           type: string
+     oneOf:
+       - required:
+          - text
+          - asset_types
+       - required:
+          - text
+          - asset_ids

spec/content/manifest.spec.yml

@@ -0,0 +1,70 @@
+##
+## Describes the specification for the content package's main manifest.yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  additionalProperties: false
+  definitions:
+    discovery:
+      description: >
+        Description of the data this package can be used with. It can be used to discover
+        the package from elements in the existing data.
+      type: object
+      additionalProperties: false
+      properties:
+        fields:
+          description: Description of the fields this package can be used with.
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                description: Name of the field.
+                type: string
+  properties:
+    format_version:
+      description: The version of the package specification format used by this package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    name:
+      description: The name of the package.
+      type: string
+      pattern: '^[a-z0-9_]+$'
+      examples:
+      - apache
+    title:
+      $ref: "../integration/manifest.spec.yml#/definitions/title"
+    description:
+      $ref: "../integration/manifest.spec.yml#/definitions/description"
+    version:
+      description: The version of the package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    source:
+      $ref: "../integration/manifest.spec.yml#/definitions/source"
+    type:
+      description: The type of package.
+      type: string
+      enum:
+      - content
+      examples:
+      - content
+    categories:
+      $ref: "../integration/manifest.spec.yml#/definitions/categories"
+    conditions:
+      $ref: "../integration/manifest.spec.yml#/definitions/conditions"
+    discovery:
+      $ref: "#/definitions/discovery"
+    icons:
+      $ref: "../integration/manifest.spec.yml#/definitions/icons"
+    screenshots:
+      $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
+    owner:
+      $ref: "../integration/manifest.spec.yml#/definitions/owner"
+  required:
+  - format_version
+  - name
+  - title
+  - description
+  - version
+  - type
+  - owner

spec/content/spec.yml

@@ -0,0 +1,61 @@
+##
+## Entrypoint of "content packages" specification.
+## The specification is considered "beta" at the moment, so it may change until we release it as GA.
+##
+## Describes the folders and files that make up a package.
+##
+spec:
+  additionalContents: false
+  totalContentsLimit: 65535
+  totalSizeLimit: 50MB
+  sizeLimit: 30MB
+  configurationSizeLimit: 5MB
+  relativePathSizeLimit: 3MB
+  fieldsPerDataStreamLimit: 2048
+  release: beta
+  contents:
+  - description: The main package manifest file
+    type: file
+    contentMediaType: "application/x-yaml"
+    sizeLimit: 5MB
+    name: "manifest.yml"
+    required: true
+    $ref: "./manifest.spec.yml"
+  - description: The package's CHANGELOG file
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "changelog.yml"
+    required: true
+    $ref: "../integration/changelog.spec.yml"
+  - description: The package's license file
+    type: file
+    contentMediaType: "text/plain"
+    name: "LICENSE.txt"
+    required: false
+  - description: Folder containing documentation for the package
+    type: folder
+    name: docs
+    required: true
+    $ref: "../integration/docs/spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "../integration/img/spec.yml"
+  - description: Folder containing Kibana assets provided by the package
+    type: folder
+    name: kibana
+    required: false
+    $ref: "./kibana/spec.yml"
+  - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package.
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "validation.yml"
+    required: false
+    $ref: "../integration/validation.spec.yml"
+
+versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/contents" # Package type not available before this version.

spec/input/spec.yml

@@ -1,6 +1,5 @@
 ##
 ## Entrypoint of "input packages" specification.
-## The specification is considered "beta" at the moment, so it may change until we release it as GA.
 ##
 ## Describes the folders and files that make up a package.
 ##

spec/integration/docs/spec.yml

@@ -1,13 +1,13 @@
-  spec:
-    additionalContents: false
-    contents:
-    - description: Main README file
-      type: file
-      contentMediaType: "text/markdown"
-      name: "README.md"
-      required: true
-    - description: Other README files (can be used by policy templates)
-      type: file
-      contentMediaType: "text/markdown"
-      pattern: '^.+.md'
-      required: false
+spec:
+  additionalContents: false
+  contents:
+  - description: Main README file
+    type: file
+    contentMediaType: "text/markdown"
+    name: "README.md"
+    required: true
+  - description: Other README files (can be used by policy templates)
+    type: file
+    contentMediaType: "text/markdown"
+    pattern: '^.+.md'
+    required: false

spec/integration/img/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true # TODO: Add definition for this dir?

test/packages/good_content/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.0
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/777

test/packages/good_content/docs/README.md

@@ -0,0 +1 @@
+# Reference package of content type