Add support for Kibana knowledge base entry assets

code/go/internal/validator/content.go

@@ -25,6 +25,7 @@ func validateContentType(fsys fs.FS, path string, contentType spectypes.ContentT
 			}
 		}
 	case "application/json":
+	case "application/x-ndjson":
 	case "text/markdown":
 	case "text/plain":
 	default:

code/go/pkg/validator/validator_test.go

@@ -44,6 +44,7 @@ func TestValidateFile(t *testing.T) {
 		"custom_ilm_policy":                  {},
 		"profiling_symbolizer":               {},
 		"logs_synthetic_mode":                {},
+		"knowledge_base":                     {},
 		"bad_additional_content": {
 			"bad-bad",
 			[]string{
@@ -227,6 +228,13 @@ func TestValidateFile(t *testing.T) {
 				`field data_stream.vars.data_stream.dataset: Does not match pattern '^[a-zA-Z0-9]+[a-zA-Z0-9\._]*$'`,
 			},
 		},
+		"bad_knowledge_base": {
+			"kibana/knowledge_base_entry/foo/manifest.yml",
+			[]string{
+				`field (root): index is required`,
+				`field (root): Additional property unknown is not allowed`,
+			},
+		},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -7,6 +7,12 @@
     - description: Add support for content packages.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/777
+    - description: Add support for "Kibana knowledge base entry" assets.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807
+    - description: Add support for semantic_text field definition
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807
 - version: 3.3.0-next
   changes:
     - description: Add support for `slo` assets.

spec/content/kibana/spec.yml

@@ -29,3 +29,8 @@ spec:
       pattern: '^{PACKAGE_NAME}-.+\.json$'
       forbiddenPatterns:
         - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+  - description: Folder containing Kibana knowledge base entries
+    type: folder
+    name: knowledge_base_entry
+    required: false
+    $ref: "../../integration/kibana/knowledge_base_entry/spec.yml"

spec/integration/data_stream/fields/fields.spec.yml

@@ -91,6 +91,7 @@ spec:
         - version
         - unsigned_long
         - counted_keyword
+        - semantic_text
 
       description:
         description: Short description of field
@@ -423,6 +424,12 @@ spec:
         type: boolean
         default: true
 
+      inference_id:
+        description: >
+          For semantic_text fields, this specifies the id of the inference
+          endpoint associated with the field
+        type: string
+
       # Conditional properties.
       default_metric: true
       metrics: true
@@ -590,12 +597,30 @@ spec:
                 - object
           required:
             - type
+      - if:
+          required:
+            - inference_id
+        then:
+          properties:
+            type:
+              enum:
+                - semantic_text
+          required:
+            - type
 
     required:
     - name
 
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/items/properties/type/enum/35" #remove semantic_text type
+      - op: remove
+        path: "/items/allOf/9" # removing inference_id when type is semantic_text
+      - op: remove
+        path: "/items/properties/inference_id" # removing inference_id field
   - before: 3.2.0
     patch:
       - op: remove

spec/integration/kibana/knowledge_base_entry/manifest.spec.yml

@@ -0,0 +1,43 @@
+##
+## Describes the specification for a Kibana knowledge base entry's manifest.yml file
+##
+spec:
+  type: object
+  additionalProperties: false
+  properties:
+    name:
+      description: The name of the knowledge base entry
+      type: string
+    description:
+      description: A description of what the knowledge base entry provides
+      type: string
+    index:
+      type: object
+      additionalProperties: false
+      properties:
+        system:
+          description: Specify whether the index should be system-managed or not
+          type: boolean
+      required:
+        - system
+    retrieval:
+      type: object
+      additionalProperties: false
+      properties:
+        syntactic_fields:
+          description: List of fields that should be used for syntactic search during retrieval.
+          type: array
+          items:
+            type: string
+        semantic_fields:
+          description: List of fields that should be used for semantic search during retrieval.
+          type: array
+          items:
+            type: string
+      required:
+        - syntactic_fields
+        - semantic_fields
+  required:
+    - name
+    - index
+    - retrieval

spec/integration/kibana/knowledge_base_entry/spec.yml

@@ -0,0 +1,33 @@
+spec:
+  additionalContents: false
+  totalContentsLimit: 50
+  contents:
+    - description: Folder containing a single knowledge base entry definition
+      type: folder
+      pattern: '^[a-z0-9_]+[a-z0-9]$'
+      required: true
+      additionalContents: false
+      contents:
+        - description: A knowledge base entry's manifest file
+          type: file
+          contentMediaType: "application/x-yaml"
+          sizeLimit: 2MB
+          name: "manifest.yml"
+          required: true
+          $ref: "./manifest.spec.yml"
+        - description: Folder containing field definitions for the knowledge base index
+          type: folder
+          name: fields
+          required: true
+          $ref: "../../data_stream/fields/spec.yml"
+        - description: Folder containing the knowledge base content files
+          type: folder
+          name: content
+          required: true
+          additionalContents: false
+          contents:
+            - description: A knowledge base entry's content file
+              type: file
+              sizeLimit: 20MB
+              contentMediaType: "application/x-ndjson"
+              pattern: '^content(-[a-z0-9]+)?\.ndjson$'

spec/integration/kibana/spec.yml

@@ -135,8 +135,17 @@ spec:
       contentMediaType: "application/json"
       pattern: '^{PACKAGE_NAME}-.+\.json$'
       forbiddenPatterns:
-        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden 
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+  - description: Folder containing Kibana knowledge base entries
+    type: folder
+    name: knowledge_base_entry
+    required: false
+    $ref: "./knowledge_base_entry/spec.yml"
 versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/contents/14" # remove kb_entry definitions
   - before: 3.3.0
     patch:
       - op: remove

test/packages/bad_knowledge_base/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

test/packages/bad_knowledge_base/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.0
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/807

test/packages/bad_knowledge_base/docs/README.md

@@ -0,0 +1 @@
+This is a template for the package README.