Merge pull request #660 from elastic/add-hidden-to-datastream

Add hidden field to data stream

CHANGELOG.md

@@ -26,7 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * Add "traces" as legal event type. [#656](https://github.com/elastic/package-registry/pull/656)
 * Add input-level `template_path` field. [#655](https://github.com/elastic/package-registry/pull/655)
-
+* Add "hidden" field to data stream. [#660] (https://github.com/elastic/package-registry/pull/660)
 ### Deprecated
 
 ### Known Issue

testdata/generated/categories-experimental.json

@@ -22,7 +22,7 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 10
+    "count": 11
   },
   {
     "id": "message_queue",

testdata/generated/categories.json

@@ -17,7 +17,7 @@
   {
     "id": "custom",
     "title": "Custom",
-    "count": 10
+    "count": 11
   },
   {
     "id": "message_queue",

testdata/generated/package/hidden/1.0.0/index.json

@@ -0,0 +1,42 @@
+{
+  "name": "hidden",
+  "title": "Hidden",
+  "version": "1.0.0",
+  "release": "beta",
+  "description": "This is the hidden integration",
+  "type": "solution",
+  "download": "/epr/hidden/hidden-1.0.0.zip",
+  "path": "/package/hidden/1.0.0",
+  "format_version": "1.0.0",
+  "readme": "/package/hidden/1.0.0/docs/README.md",
+  "license": "basic",
+  "categories": [
+    "custom"
+  ],
+  "conditions": {
+    "kibana.version": "\u003e=7.0.0"
+  },
+  "assets": [
+    "/package/hidden/1.0.0/manifest.yml",
+    "/package/hidden/1.0.0/docs/README.md",
+    "/package/hidden/1.0.0/data_stream/hidden/manifest.yml",
+    "/package/hidden/1.0.0/data_stream/hidden/fields/base-fields.yml",
+    "/package/hidden/1.0.0/data_stream/hidden/fields/some_fields.yml"
+  ],
+  "data_streams": [
+    {
+      "type": "metrics",
+      "dataset": "hidden.hidden",
+      "hidden": true,
+      "title": "Hidden data stream and ilm policy overrride",
+      "release": "experimental",
+      "package": "hidden",
+      "elasticsearch": {
+        "index_template.mappings": {
+          "dynamic": false
+        }
+      },
+      "path": "hidden"
+    }
+  ]
+}

testdata/generated/search-all.json

@@ -59,6 +59,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "input_level_templates",
     "title": "Input level templates",

testdata/generated/search-category-custom.json

@@ -19,6 +19,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "input_level_templates",
     "title": "Input level templates",

testdata/generated/search-kibana721.json

@@ -49,6 +49,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "longdocs",
     "title": "Long Docs",

testdata/generated/search-package-experimental.json

@@ -69,6 +69,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "input_level_templates",
     "title": "Input level templates",

testdata/generated/search-package-internal.json

@@ -49,6 +49,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "input_level_templates",
     "title": "Input level templates",

testdata/generated/search.json

@@ -49,6 +49,16 @@
     "download": "/epr/foo/foo-1.0.0.zip",
     "path": "/package/foo/1.0.0"
   },
+  {
+    "name": "hidden",
+    "title": "Hidden",
+    "version": "1.0.0",
+    "release": "beta",
+    "description": "This is the hidden integration",
+    "type": "solution",
+    "download": "/epr/hidden/hidden-1.0.0.zip",
+    "path": "/package/hidden/1.0.0"
+  },
   {
     "name": "input_level_templates",
     "title": "Input level templates",

testdata/package/hidden/1.0.0/data_stream/hidden/fields/base-fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: >
+    Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: >
+    Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: >
+    Data stream namespace.
+- name: "@timestamp"
+  type: date
+  description: >
+    Event timestamp.

testdata/package/hidden/1.0.0/data_stream/hidden/fields/some_fields.yml

@@ -0,0 +1,15 @@
+
+- name: source
+  title: Source
+  group: 2
+  type: group
+  fields:
+    - name: geo.city_name
+      level: core
+      type: keyword
+      description: City name.
+      ignore_above: 1024
+- name: foobar
+  type: text
+  description: A field with a pattern defined
+  pattern: '^[a-zA-Z]$'

testdata/package/hidden/1.0.0/data_stream/hidden/manifest.yml

@@ -0,0 +1,6 @@
+title: Hidden data stream and ilm policy overrride
+type: metrics
+hidden: true
+elasticsearch:
+  index_template.mappings:
+    dynamic: false

testdata/package/hidden/1.0.0/manifest.yml

@@ -0,0 +1,14 @@
+format_version: 1.0.0
+
+name: hidden
+description: This is the hidden integration
+version: 1.0.0
+title: Hidden
+categories: ["custom"]
+type: solution
+release: beta
+
+conditions:
+  kibana:
+    version: ">=7.0.0"
+

util/data_stream.go

@@ -38,6 +38,7 @@ type DataStream struct {
 	// Name and type of the data stream. This is linked to data_stream.dataset and data_stream.type fields.
 	Type    string `config:"type" json:"type" validate:"required"`
 	Dataset string `config:"dataset" json:"dataset,omitempty" yaml:"dataset,omitempty"`
+	Hidden  bool   `config:"hidden" json:"hidden,omitempty" yaml:"hidden,omitempty"`
 
 	Title   string `config:"title" json:"title" validate:"required"`
 	Release string `config:"release" json:"release"`