Merge branch 'master' into update-indices

dev_docs/key_concepts/anatomy_of_a_plugin.mdx

@@ -75,7 +75,7 @@ plugins/
 - preboot plugins are bootstrapped to prepare the environment before Kibana starts.
 - standard plugins define Kibana functionality while Kibana is running.
 
-`owner` - [Required] Help users of your plugin know who manages this plugin and how to get in touch. This is required for internal plugins. `Owner.name` should be the name of the team that manages this plugin. This should match the team that owns this code in the [CODEOWNERS](https://github.com/elastic/kibana/blob/master/.github/CODEOWNERS) file (however, this is not currently enforced). Internal teams should also use a [GitHub team alias](https://github.com/orgs/elastic/teams) for `owner.githubTeam`. While many teams can contribute to a plugin, only a single team should be the primary owner.
+`owner` - [Required] Help users of your plugin know who manages this plugin and how to get in touch. For internal developers, `Owner.name` should be the name of the team that manages this plugin. This should match the team that owns this code in the [CODEOWNERS](https://github.com/elastic/kibana/blob/master/.github/CODEOWNERS) file (however, this is not currently enforced). Internal teams should also use a [GitHub team alias](https://github.com/orgs/elastic/teams) for `owner.githubTeam`. This value is used to create a link to `https://github.com/orgs/elastic/teams/${githubTeam}`, so leave the `elastic/` prefix should be left out. While many teams can contribute to a plugin, only a single team should be the primary owner.
 
 `description` - [Required] Give your plugin a description to help other developers understand what it does. This is required for internal plugins.
 

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -26,7 +26,7 @@ Should never be used in code outside of Core but is exported for documentation p
 | [id](./kibana-plugin-core-server.pluginmanifest.id.md) | <code>PluginName</code> | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. |
 | [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | <code>string</code> | The version of Kibana the plugin is compatible with, defaults to "version". |
 | [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |
-| [owner](./kibana-plugin-core-server.pluginmanifest.owner.md) | <code>{</code><br/><code> readonly name: string;</code><br/><code> readonly githubTeam?: string;</code><br/><code> }</code> | TODO: make required once all internal plugins have this specified. |
+| [owner](./kibana-plugin-core-server.pluginmanifest.owner.md) | <code>{</code><br/><code> readonly name: string;</code><br/><code> readonly githubTeam?: string;</code><br/><code> }</code> | |
 | [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) | <code>readonly string[]</code> | List of plugin ids that this plugin's UI code imports modules from that are not in <code>requiredPlugins</code>. |
 | [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 | [server](./kibana-plugin-core-server.pluginmanifest.server.md) | <code>boolean</code> | Specifies whether plugin includes some server-side specific functionality. |

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.owner.md

@@ -4,12 +4,10 @@
 
 ## PluginManifest.owner property
 
-TODO: make required once all internal plugins have this specified.
-
 <b>Signature:</b>
 
 ```typescript
-readonly owner?: {
+readonly owner: {
  readonly name: string;
  readonly githubTeam?: string;
  };

docs/maps/index.asciidoc

@@ -75,6 +75,7 @@ Search across the layers in your map to focus in on just the data you want. Comb
 
 include::maps-getting-started.asciidoc[]
 include::asset-tracking-tutorial.asciidoc[]
+include::reverse-geocoding-tutorial.asciidoc[]
 include::heatmap-layer.asciidoc[]
 include::tile-layer.asciidoc[]
 include::vector-layer.asciidoc[]

docs/maps/reverse-geocoding-tutorial.asciidoc

@@ -0,0 +1,182 @@
+[role="xpack"]
+[[reverse-geocoding-tutorial]]
+== Map custom regions with reverse geocoding
+
+*Maps* comes with https://maps.elastic.co/#file[predefined regions] that allow you to quickly visualize regions by metrics. *Maps* also offers the ability to map your own regions. You can use any region data you'd like, as long as your source data contains an identifier for the corresponding region. 
+
+But how can you map regions when your source data does not contain a region identifier? This is where reverse geocoding comes in. Reverse geocoding is the process of assigning a region identifer to a feature based on its location.
+
+In this tutorial, you’ll use reverse geocoding to visualize United States Census Bureau Combined Statistical Area (CSA) regions by web traffic.
+
+You’ll learn to:
+
+- Upload custom regions.
+- Reverse geocode with the {es} {ref}/enrich-processor.html[enrich processor].
+- Create a map and visualize CSA regions by web traffic.
+
+When you complete this tutorial, you’ll have a map that looks like this:
+
+[role="screenshot"]
+image::maps/images/reverse-geocoding-tutorial/csa_regions_by_web_traffic.png[]
+
+
+[float]
+=== Step 1: Index web traffic data
+GeoIP is a common way of transforming an IP address to a longitude and latitude. GeoIP is roughly accurate on the city level globally and neighborhood level in selected countries. It’s not as good as an actual GPS location from your phone, but it’s much more precise than just a country, state, or province.
+
+You’ll use the <<get-started, web logs sample data set>> that comes with Kibana for this tutorial. Web logs sample data set has longitude and latitude. If your web log data does not contain longitude and latitude, use {ref}/geoip-processor.html[GeoIP processor] to transform an IP address into a {ref}/geo-point.html[geo_point] field.
+
+To install web logs sample data set:
+
+. On the home page, click *Try sample data*.
+. On the *Sample web logs* card, click *Add data*.
+
+
+[float]
+=== Step 2: Index Combined Statistical Area (CSA) regions
+GeoIP level of detail is very useful for driving decision-making. For example, say you want to spin up a marketing campaign based on the locations of your users or show executive stakeholders which metro areas are experiencing an uptick of traffic.
+
+That kind of scale in the United States is often captured with what the Census Bureau calls the Combined Statistical Area (CSA). Combined Statistical Area is roughly equivalent with how people intuitively think of which urban area they live in. It does not necessarily coincide with state or city boundaries.
+
+CSAs generally share the same telecom providers and ad networks. New fast food franchises expand to a CSA rather than a particular city or municipality. Basically, people in the same CSA shop in the same IKEA.
+
+To get the CSA boundary data:
+
+. Download the https://www.census.gov/geographies/mapping-files/time-series/geo/carto-boundary-file.html[Cartographic Boundary shapefile (.shp)] from the Census Bureau’s website.
+. To use the data in Kibana, convert it to GeoJSON format. Follow this https://gist.github.com/YKCzoli/b7f5ff0e0f641faba0f47fa5d16c4d8d[helpful tutorial] to use QGIS to convert the Cartographic Boundary shapefile to GeoJSON. Or, download a https://raw.githubusercontent.com/elastic/examples/master/blog/reverse-geocoding/csba.json[prebuilt GeoJSON version].
+
+Once you have your GeoJSON file:
+
+. Open the main menu, and click *Maps*.
+. Click *Create map*.
+. Click *Add layer*.
+. Click *Upload GeoJSON*.
+. Use the file chooser to import the CSA GeoJSON file.
+. Set index name to *csa* and click *Import file*.
+. When importing is complete, click *Add as document layer*.
+. Add Tooltip fields:
+.. Click *+ Add* to open field select.
+.. Select *NAME*, *GEOID*, and *AFFGEOID*.
+.. Click *Add*.
+. Click *Save & close*.
+
+Looking at the map, you get a sense of what constitutes a metro area in the eyes of the Census Bureau.
+
+[role="screenshot"]
+image::maps/images/reverse-geocoding-tutorial/csa_regions.jpeg[]
+
+[float]
+=== Step 3: Reverse geocoding
+To visualize CSA regions by web log traffic, the web log traffic must contain a CSA region identifier. You'll use {es} {ref}/enrich-processor.html[enrich processor] to add CSA region identifiers to the web logs sample data set. You can skip this step if your source data already contains region identifiers.
+
+. Open the main menu, then click *Dev Tools*.
+. In *Console*, create a {ref}/geo-match-enrich-policy-type.html[geo_match enrichment policy]:
++
+[source,js]
+----------------------------------
+PUT /_enrich/policy/csa_lookup 
+{ 
+ "geo_match": { 
+ "indices": "csa", 
+ "match_field": "coordinates", 
+ "enrich_fields": [ "GEOID", "NAME"] 
+ } 
+}
+----------------------------------
+
+. To initialize the policy, run:
++
+[source,js]
+----------------------------------
+POST /_enrich/policy/csa_lookup/_execute
+----------------------------------
+
+. To create a ingest pipeline, run:
++
+[source,js]
+----------------------------------
+PUT _ingest/pipeline/lonlat-to-csa 
+{ 
+ "description": "Reverse geocode longitude-latitude to combined statistical area", 
+ "processors": [ 
+ { 
+ "enrich": { 
+ "field": "geo.coordinates", 
+ "policy_name": "csa_lookup", 
+ "target_field": "csa", 
+ "ignore_missing": true, 
+ "ignore_failure": true, 
+ "description": "Lookup the csa identifier" 
+ } 
+ }, 
+ { 
+ "remove": { 
+ "field": "csa.coordinates", 
+ "ignore_missing": true, 
+ "ignore_failure": true, 
+ "description": "Remove the shape field" 
+ } 
+ } 
+ ] 
+}
+----------------------------------
+
+. To update your existing data, run:
++
+[source,js]
+----------------------------------
+POST kibana_sample_data_logs/_update_by_query?pipeline=lonlat-to-csa
+----------------------------------
+
+. To run the pipeline on new documents at ingest, run:
++
+[source,js]
+----------------------------------
+PUT kibana_sample_data_logs/_settings 
+{ 
+ "index": { 
+ "default_pipeline": "lonlat-to-csa" 
+ } 
+}
+----------------------------------
+
+. Open the main menu, and click *Discover*.
+. Set the index pattern to *kibana_sample_data_logs*.
+. Open the <<set-time-filter, time filter>>, and set the time range to the last 30 days.
+. Scan through the list of *Available fields* until you find the `csa.GEOID` field. You can also search for the field by name.
+. Click image:images/reverse-geocoding-tutorial/add-icon.png[Add icon] to toggle the field into the document table.
+. Find the 'csa.NAME' field and add it to your document table.
+
+Your web log data now contains `csa.GEOID` and `csa.NAME` fields from the matching *csa* region. Web log traffic not contained in a CSA region does not have values for `csa.GEOID` and `csa.NAME` fields.
+
+[role="screenshot"]
+image::maps/images/reverse-geocoding-tutorial/discover_enriched_web_log.png[]
+
+[float]
+=== Step 4: Visualize Combined Statistical Area (CSA) regions by web traffic
+Now that our web traffic contains CSA region identifiers, you'll visualize CSA regions by web traffic.
+
+. Open the main menu, and click *Maps*.
+. Click *Create map*.
+. Click *Add layer*.
+. Click *Choropleth*.
+. For *Boundaries source*: 
+.. Select *Points, lines, and polygons from Elasticsearch*.
+.. Set *Index pattern* to *csa*.
+.. Set *Join field* to *GEOID*.
+. For *Statistics source*:
+.. Set *Index pattern* to *kibana_sample_data_logs*.
+.. Set *Join field* to *csa.GEOID.keyword*.
+. Click *Add layer*.
+. Scroll to *Layer Style* and Set *Label* to *Fixed*.
+. Click *Save & close*.
+. *Save* the map.
+.. Give the map a title.
+.. Under *Add to dashboard*, select *None*.
+.. Click *Save and add to library*.
+
+[role="screenshot"]
+image::maps/images/reverse-geocoding-tutorial/csa_regions_by_web_traffic.png[]
+
+Congratulations! You have completed the tutorial and have the recipe for visualizing custom regions. You can now try replicating this same analysis with your own data.
+

docs/user/alerting/alerting-setup.asciidoc

@@ -61,9 +61,13 @@ Rules and connectors are isolated to the {kib} space in which they were created.
 [[alerting-authorization]]
 === Authorization
 
-Rules, including all background detection and the actions they generate are authorized using an <<api-keys, API key>> associated with the last user to edit the rule. Upon creating or modifying a rule, an API key is generated for that user, capturing a snapshot of their privileges at that moment in time. The API key is then used to run all background tasks associated with the rule including detection checks and executing actions. 
+Rules are authorized using an <<api-keys, API key>> associated with the last user to edit the rule. This API key captures a snapshot of the user's privileges at the time of edit and is subsequently used to run all background tasks associated with the rule, including condition checks, like {es} queries, and action executions. The following rule actions will re-generate the API key:
+
+* Creating a rule
+* Enabling a disabled rule
+* Updating a rule
 
 [IMPORTANT]
 ==============================================
-If a rule requires certain privileges to run, such as index privileges, keep in mind that if a user without those privileges updates the rule, the rule will no longer function.  
+If a rule requires certain privileges, such as index privileges, to run, and a user without those privileges updates, disables, or re-enables the rule, the rule will no longer function. Conversely, if a user with greater or administrator privileges modifies the rule, it will begin running with increased privileges.
 ==============================================

package.json

@@ -637,6 +637,7 @@
  "@types/testing-library__jest-dom": "^5.9.5",
  "@types/testing-library__react-hooks": "^4.0.0",
  "@types/tinycolor2": "^1.4.1",
+ "@types/tough-cookie": "^4.0.1",
  "@types/type-detect": "^4.0.1",
  "@types/use-resize-observer": "^6.0.0",
  "@types/uuid": "^3.4.4",
@@ -828,6 +829,7 @@
  "terminal-link": "^2.1.1",
  "terser": "^5.7.1",
  "terser-webpack-plugin": "^2.1.2",
+ "tough-cookie": "^4.0.0",
  "ts-loader": "^7.0.5",
  "ts-morph": "^9.1.0",
  "tsd": "^0.13.1",

packages/kbn-dev-utils/src/plugins/parse_kibana_platform_plugin.ts

@@ -29,8 +29,7 @@ interface Manifest {
  server: boolean;
  kibanaVersion: string;
  version: string;
- // TODO: make this required.
- owner?: {
+ owner: {
  // Internally, this should be a team name.
  name: string;
  // All internally owned plugins should have a github team specified that can be pinged in issues, or used to look up
@@ -64,6 +63,12 @@ export function parseKibanaPlatformPlugin(manifestPath: string): KibanaPlatformP
  throw new TypeError('expected new platform plugin manifest to have a string version');
  }
 
+ if (!manifest.owner || typeof manifest.owner.name !== 'string') {
+ throw new TypeError(
+ `Expected plugin ${manifest.id} manifest to have an owner with name specified (${manifestPath})`
+ );
+ }
+
  return {
  directory: Path.dirname(manifestPath),
  manifestPath,

packages/kbn-docs-utils/src/api_docs/mdx/write_plugin_mdx_docs.ts

@@ -86,8 +86,8 @@ import ${json} from './${fileName}.json';
 ${plugin.manifest.description ?? ''}
 
 ${
- plugin.manifest.owner?.githubTeam && name
- ? `Contact [${name}](https://github.com/orgs/elastic/teams/${plugin.manifest.owner?.githubTeam}) for questions regarding this plugin.`
+ plugin.manifest.owner.githubTeam && name
+ ? `Contact [${name}](https://github.com/orgs/elastic/teams/${plugin.manifest.owner.githubTeam}) for questions regarding this plugin.`
  : name
  ? `Contact ${name} for questions regarding this plugin.`
  : ''

packages/kbn-docs-utils/src/api_docs/tests/__fixtures__/src/plugin_a/kibana.json

@@ -1,7 +1,7 @@
 {
-  "id": "pluginA",
-  "summary": "This an example plugin for testing the api documentation system",
-  "version": "kibana",
-  "serviceFolders": ["foo"]
- }
- 
+ "id": "pluginA",
+ "owner": { "name": "Kibana Tech Leads" },
+ "summary": "This an example plugin for testing the api documentation system",
+ "version": "kibana",
+ "serviceFolders": ["foo"]
+}

packages/kbn-docs-utils/src/api_docs/tests/__fixtures__/src/plugin_b/kibana.json

@@ -1,5 +1,6 @@
 {
  "id": "pluginB",
+ "owner": { "name": "Kibana Tech Leads" },
  "summary": "This an example plugin for testing the api documentation system",
  "version": "kibana"
 }

packages/kbn-docs-utils/src/api_docs/tests/kibana_platform_plugin_mock.ts

@@ -18,6 +18,9 @@ export function getKibanaPlatformPlugin(id: string, dir?: string): KibanaPlatfor
  server: true,
  kibanaVersion: '1',
  version: '1',
+ owner: {
+ name: 'Kibana Core',
+ },
  serviceFolders: [],
  requiredPlugins: [],
  requiredBundles: [],

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json

@@ -2,5 +2,8 @@
  "id": "bar",
  "ui": true,
  "requiredBundles": ["foo"],
- "version": "8.0.0"
+ "version": "8.0.0",
+ "owner": {
+ "name": "Operations"
+ }
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo/kibana.json

@@ -1,5 +1,8 @@
 {
  "id": "foo",
+ "owner": {
+ "name": "Operations"
+ },
  "ui": true,
  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/nested/baz/kibana.json

@@ -1,4 +1,7 @@
 {
  "id": "baz",
+ "owner": {
+ "name": "Operations"
+ },
  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz/kibana.json

@@ -1,4 +1,7 @@
 {
  "id": "test_baz",
+ "owner": {
+ "name": "Operations"
+ },
  "version": "8.0.0"
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/x-pack/baz/kibana.json

@@ -1,5 +1,6 @@
 {
  "id": "baz",
+ "owner": { "name": "Operations", "githubTeam": "kibana-operations" },
  "ui": true,
  "version": "8.0.0"
 }