Merge branch 'add-values-definition' of github.com:drewdaemon/kibana …

…into add-values-definition

.buildkite/scripts/steps/code_coverage/ingest.sh

@@ -3,15 +3,16 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/common/vault_fns.sh
 source .buildkite/scripts/steps/code_coverage/util.sh
 
 export CODE_COVERAGE=1
-echo "--- Reading Kibana stats cluster creds from vault"
-USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)"
+echo "--- Reading Kibana coverage creds from vault"
+USER_FROM_VAULT="$(vault_get coverage/elasticsearch username)"
 export USER_FROM_VAULT
-PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)"
+PASS_FROM_VAULT="$(vault_get coverage/elasticsearch password)"
 export PASS_FROM_VAULT
-HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
+HOST_FROM_VAULT="$(vault_get coverage/elasticsearch host)"
 export HOST_FROM_VAULT
 TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
 export TIME_STAMP

docs/management/connectors/action-types/cases-webhook.asciidoc

@@ -53,6 +53,7 @@ image::management/connectors/images/cases-webhook-connector-comments.png[Add cas
 
 Add HTTP header::
 A set of key-value pairs sent as headers with the request URLs for the create case, update case, get case, and create comment methods.
+For example, set `Content-Type` to the appropriate media type for your requests.
 
 Create case method::
 The REST API HTTP request method to create a case in the third-party system: `post`(default), `put`, or `patch`.

docs/management/connectors/action-types/webhook.asciidoc

@@ -34,7 +34,9 @@ Authentication::
 The authentication type: none, basic, or SSL.
 If you choose basic authentication, you must provide a user name and password.
 If you choose SSL authentication, you must provide SSL server certificate authentication data in a CRT and key file format or a PFX file format. You can also optionally provide a passphrase if the files are password-protected.
-HTTP headers::   A set of key-value pairs sent as headers with the request.
+HTTP headers::
+A set of key-value pairs sent as headers with the request.
+For example, set `Content-Type` to the appropriate media type for your requests.
 Certificate authority::
 A certificate authority (CA) that the connector can trust, for example to sign and validate server certificates.
 This option is available for all authentication types.

package.json

@@ -1519,6 +1519,7 @@
     "@types/vinyl-fs": "^3.0.2",
     "@types/watchpack": "^1.1.5",
     "@types/webpack": "^4.41.3",
+    "@types/webpack-bundle-analyzer": "^4.7.0",
     "@types/webpack-env": "^1.15.3",
     "@types/webpack-merge": "^4.1.5",
     "@types/webpack-sources": "^0.1.4",

packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts

@@ -180,7 +180,7 @@ export const HASH_TO_VERSION_MAP = {
   'infrastructure-monitoring-log-view|c50526fc6040c5355ed027d34d05b35c': '10.0.0',
   'infrastructure-ui-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
   'ingest_manager_settings|b91ffb075799c78ffd7dbd51a279c8c9': '10.1.0',
-  'ingest-agent-policies|20768dc7ce5eced3eb309e50d8a6cf76': '10.0.0',
+  'ingest-agent-policies|0fd93cd11c019b118e93a9157c22057b': '10.1.0',
   'ingest-download-sources|0b0f6828e59805bd07a650d80817c342': '10.0.0',
   'ingest-outputs|b1237f7fdc0967709e75d65d208ace05': '10.6.0',
   'ingest-package-policies|a1a074bad36e68d54f98d2158d60f879': '10.0.0',

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/README.md

@@ -456,7 +456,7 @@ Once transformed we use an index operation to overwrite the outdated document wi
 
 ### Next action
 
-`checkTargetMappings`
+`checkTargetTypesMappings`
 
 Compare the calculated mappings' hashes against those stored in the `<index>.mappings._meta`.
 

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts

@@ -9,7 +9,7 @@
 import * as Either from 'fp-ts/lib/Either';
 import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
 import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server';
-import { checkTargetMappings } from './check_target_mappings';
+import { checkTargetTypesMappings } from './check_target_mappings';
 import { getBaseMappings } from '../core';
 
 const indexTypes = ['type1', 'type2', 'type3'];
@@ -60,14 +60,14 @@ const appMappings: IndexMapping = {
   },
 };
 
-describe('checkTargetMappings', () => {
+describe('checkTargetTypesMappings', () => {
   beforeEach(() => {
     jest.clearAllMocks();
   });
 
   describe('when index mappings are missing required properties', () => {
     it("returns 'index_mappings_incomplete' if index mappings are not defined", async () => {
-      const task = checkTargetMappings({
+      const task = checkTargetTypesMappings({
         indexTypes,
         appMappings,
         latestMappingsVersions,
@@ -78,7 +78,7 @@ describe('checkTargetMappings', () => {
     });
 
     it("returns 'index_mappings_incomplete' if index mappings do not define _meta", async () => {
-      const task = checkTargetMappings({
+      const task = checkTargetTypesMappings({
         indexTypes,
         appMappings,
         indexMappings: {
@@ -93,7 +93,7 @@ describe('checkTargetMappings', () => {
     });
 
     it("returns 'index_mappings_incomplete' if index mappings do not define migrationMappingPropertyHashes nor mappingVersions", async () => {
-      const task = checkTargetMappings({
+      const task = checkTargetTypesMappings({
         indexTypes,
         appMappings,
         indexMappings: {
@@ -109,7 +109,7 @@ describe('checkTargetMappings', () => {
     });
 
     it("returns 'index_mappings_incomplete' if index mappings define a different value for 'dynamic' property", async () => {
-      const task = checkTargetMappings({
+      const task = checkTargetTypesMappings({
         indexTypes,
         appMappings,
         indexMappings: {
@@ -128,7 +128,7 @@ describe('checkTargetMappings', () => {
   describe('when index mappings have all required properties', () => {
     describe('when some core properties (aka root fields) have changed', () => {
       it('returns the list of fields that have changed', async () => {
-        const task = checkTargetMappings({
+        const task = checkTargetTypesMappings({
           indexTypes,
           appMappings,
           indexMappings: {
@@ -149,8 +149,9 @@ describe('checkTargetMappings', () => {
         const result = await task();
         expect(result).toEqual(
           Either.left({
-            type: 'root_fields_changed' as const,
-            updatedFields: ['references'],
+            type: 'types_changed' as const,
+            // types are flagged as changed cause we have not provided a hashToVersionMap
+            updatedTypes: ['type1', 'type2'],
           })
         );
       });
@@ -159,8 +160,8 @@ describe('checkTargetMappings', () => {
     describe('when core properties have NOT changed', () => {
       describe('when index mappings ONLY contain the legacy hashes', () => {
         describe('and legacy hashes match the current model versions', () => {
-          it('returns a compared_mappings_match response', async () => {
-            const task = checkTargetMappings({
+          it('returns a types_match response', async () => {
+            const task = checkTargetTypesMappings({
               indexTypes,
               appMappings,
               indexMappings: legacyMappings,
@@ -175,15 +176,15 @@ describe('checkTargetMappings', () => {
             const result = await task();
             expect(result).toEqual(
               Either.right({
-                type: 'compared_mappings_match' as const,
+                type: 'types_match' as const,
               })
             );
           });
         });
 
         describe('and legacy hashes do NOT match the current model versions', () => {
           it('returns the list of updated SO types', async () => {
-            const task = checkTargetMappings({
+            const task = checkTargetTypesMappings({
               indexTypes,
               appMappings,
               indexMappings: legacyMappings,
@@ -207,8 +208,8 @@ describe('checkTargetMappings', () => {
 
       describe('when index mappings contain the mappingVersions', () => {
         describe('and mappingVersions match', () => {
-          it('returns a compared_mappings_match response', async () => {
-            const task = checkTargetMappings({
+          it('returns a types_match response', async () => {
+            const task = checkTargetTypesMappings({
               indexTypes,
               appMappings,
               indexMappings: {
@@ -228,15 +229,15 @@ describe('checkTargetMappings', () => {
             const result = await task();
             expect(result).toEqual(
               Either.right({
-                type: 'compared_mappings_match' as const,
+                type: 'types_match' as const,
               })
             );
           });
         });
 
         describe('and mappingVersions do NOT match', () => {
           it('returns the list of updated SO types', async () => {
-            const task = checkTargetMappings({
+            const task = checkTargetTypesMappings({
               indexTypes,
               appMappings,
               indexMappings: {

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts

@@ -9,10 +9,10 @@ import * as Either from 'fp-ts/lib/Either';
 import * as TaskEither from 'fp-ts/lib/TaskEither';
 
 import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal';
-import { getUpdatedRootFields, getUpdatedTypes } from '../core/compare_mappings';
+import { getUpdatedTypes } from '../core/compare_mappings';
 
 /** @internal */
-export interface CheckTargetMappingsParams {
+export interface CheckTargetTypesMappingsParams {
   indexTypes: string[];
   indexMappings?: IndexMapping;
   appMappings: IndexMapping;
@@ -21,34 +21,31 @@ export interface CheckTargetMappingsParams {
 }
 
 /** @internal */
-export interface ComparedMappingsMatch {
-  type: 'compared_mappings_match';
-}
-
 export interface IndexMappingsIncomplete {
   type: 'index_mappings_incomplete';
 }
 
-export interface RootFieldsChanged {
-  type: 'root_fields_changed';
-  updatedFields: string[];
+/** @internal */
+export interface TypesMatch {
+  type: 'types_match';
 }
 
+/** @internal */
 export interface TypesChanged {
   type: 'types_changed';
   updatedTypes: string[];
 }
 
-export const checkTargetMappings =
+export const checkTargetTypesMappings =
   ({
     indexTypes,
     indexMappings,
     appMappings,
     latestMappingsVersions,
     hashToVersionMap = {},
-  }: CheckTargetMappingsParams): TaskEither.TaskEither<
-    IndexMappingsIncomplete | RootFieldsChanged | TypesChanged,
-    ComparedMappingsMatch
+  }: CheckTargetTypesMappingsParams): TaskEither.TaskEither<
+    IndexMappingsIncomplete | TypesChanged,
+    TypesMatch
   > =>
   async () => {
     if (
@@ -59,15 +56,6 @@ export const checkTargetMappings =
       return Either.left({ type: 'index_mappings_incomplete' as const });
     }
 
-    const updatedFields = getUpdatedRootFields(indexMappings);
-
-    if (updatedFields.length) {
-      return Either.left({
-        type: 'root_fields_changed',
-        updatedFields,
-      });
-    }
-
     const updatedTypes = getUpdatedTypes({
       indexTypes,
       indexMeta: indexMappings?._meta,
@@ -81,6 +69,6 @@ export const checkTargetMappings =
         updatedTypes,
       });
     } else {
-      return Either.right({ type: 'compared_mappings_match' as const });
+      return Either.right({ type: 'types_match' as const });
     }
   };

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts

@@ -87,7 +87,7 @@ export { synchronizeMigrators } from './synchronize_migrators';
 
 export { createIndex } from './create_index';
 
-export { checkTargetMappings } from './check_target_mappings';
+export { checkTargetTypesMappings } from './check_target_mappings';
 
 export const noop = async (): Promise<Either<never, 'noop'>> => right('noop' as const);
 
@@ -108,11 +108,7 @@ import type { UnknownDocsFound } from './check_for_unknown_docs';
 import type { IncompatibleClusterRoutingAllocation } from './initialize_action';
 import type { ClusterShardLimitExceeded } from './create_index';
 import type { SynchronizationFailed } from './synchronize_migrators';
-import type {
-  IndexMappingsIncomplete,
-  RootFieldsChanged,
-  TypesChanged,
-} from './check_target_mappings';
+import type { IndexMappingsIncomplete, TypesChanged } from './check_target_mappings';
 
 export type {
   CheckForUnknownDocsParams,
@@ -187,7 +183,6 @@ export interface ActionErrorTypeMap {
   es_response_too_large: EsResponseTooLargeError;
   synchronization_failed: SynchronizationFailed;
   index_mappings_incomplete: IndexMappingsIncomplete;
-  root_fields_changed: RootFieldsChanged;
   types_changed: TypesChanged;
   operation_not_supported: OperationNotSupported;
 }

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts

@@ -10,10 +10,18 @@ import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-ba
 import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings';
 
 /**
- * Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because
- * actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes
- * match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana
- * doesn't know about or expect, without triggering continual migrations.
+ * Diffs the stored vs app mappings.
+ * On one hand, it compares changes in root fields, by deep comparing the actual mappings.
+ * On the other hand, it compares changes in SO types mappings:
+ * Historically, this comparison was done using md5 hashes.
+ * Currently, and in order to be FIPS compliant, this has been replaced by comparing model versions.
+ * The `getUpdatedTypes` uses a map to handle the transition md5 => modelVersion
+
+ * @param indexMappings The mappings stored in the SO index
+ * @param appMappings The current Kibana mappings, computed from the typeRegistry
+ * @param indexTypes A list of the SO types that are bound to the SO index
+ * @param latestMappingsVersions A map containing the latest version in which each type has updated its mappings
+ * @param hashToVersionMap Map that holds md5 => modelVersion equivalence, to smoothly transition away from hashes
  */
 export function diffMappings({
   indexMappings,