Add new "references" attribute to saved objects for relationships (#2…

…8199)

* Add new references attribute to saved objects

* Add dual support for dashboard export API

* Use new relationships API supporting legacy relationships extraction

* Code cleanup

* Fix style and CI error

* Add missing spaces test for findRelationships

* Convert collect_references_deep to typescript

* Add missing trailing commas

* Fix broken test by making saved object API consistently return references

* Fix broken api integration tests

* Add comment about the two TS types for saved object

* Only return title from the attributes returned in findRelationships

* Fix broken test

* Add missing security tests

* Drop filterTypes support

* Implement references to search, dashboard, visualization, graph

* Add index pattern migration to dashboards

* Add references mapping to dashboard mppings.json

* Remove findRelationships from repository and into it's own function / file

* Apply PR feedback pt1

* Fix some failing tests

* Remove error throwing in migrations

* Add references to edit saved object screen

* Pass types to findRelationships

* [ftr] restore snapshots from master, rely on migrations to add references

* [security] remove `find_relationships` action

* remove data set modifications

* [security/savedObjectsClient] remove _getAuthorizedTypes method

* fix security & spaces tests to consider references and migrationVersion

* Add space id prefixes to es_archiver/saved_objects/spaces/data.json

* Rename referenced attributes to have a suffix of RefName

* Fix length check in scenario references doesn't exist

* Add test for inject references to not be called when references array is empty or missing

* some code cleanup

* Make migrations run on machine learning data files, fix rollup filterPath for savedSearchRefName

* fix broken test

* Fix collector.js to include references in elasticsearch response

* code cleanup pt2

* add some more tests

* fix broken tests

* updated documentation on referencedBy option for saved object client find function

* Move visualization migrations into kibana plugin

* Update docs with better description on references

* Apply PR feedback

* Fix merge

* fix tests I broke adressing PR feedback

* PR feedback pt2

docs/api/saved-objects/bulk_create.asciidoc

@@ -33,6 +33,9 @@ contains the following properties:
 `attributes` (required)::
   (object) The data to persist
 
+`references` (optional)::
+  (array) An array of objects with `name`, `id`, and `type` properties that describe the other saved objects this object references. The `name` can be used in the attributes to refer to the other saved object, but never the `id`, which may be updated automatically in the future during migrations or import/export.
+
 `version` (optional)::
   (number) Enables specifying a version
 

docs/api/saved-objects/create.asciidoc

@@ -33,6 +33,8 @@ Note: You cannot access this endpoint via the Console in Kibana.
 `attributes` (required)::
   (object) The data to persist
 
+`references` (optional)::
+  (array) An array of objects with `name`, `id`, and `type` properties that describe the other saved objects this object references. The `name` can be used in the attributes to refer to the other saved object, but never the `id`, which may be updated automatically in the future during migrations or import/export.
 
 ==== Examples
 

docs/api/saved-objects/find.asciidoc

@@ -29,6 +29,8 @@ Note: You cannot access this endpoint via the Console in Kibana.
   (array|string) The fields to return in the response
 `sort_field` (optional)::
   (string) The field on which the response will be sorted
+`has_reference` (optional)::
+  (object) Filters to objects having a relationship with the type and id combination
 
 [NOTE]
 ==============================================

docs/api/saved-objects/update.asciidoc

@@ -26,6 +26,8 @@ Note: You cannot access this endpoint via the Console in Kibana.
 `attributes` (required)::
   (object) The data to persist
 
+`references` (optional)::
+  (array) An array of objects with `name`, `id`, and `type` properties that describe the other saved objects this object references. The `name` can be used in the attributes to refer to the other saved object, but never the `id`, which may be updated automatically in the future during migrations or import/export.
 
 ==== Examples
 

src/legacy/core_plugins/kibana/mappings.json

@@ -42,7 +42,7 @@
           }
         }
       },
-      "savedSearchId": {
+      "savedSearchRefName": {
         "type": "keyword"
       },
       "title": {

src/legacy/core_plugins/kibana/migrations.js

@@ -19,9 +19,53 @@
 
 import { cloneDeep, get, omit } from 'lodash';
 
+function migrateIndexPattern(doc) {
+  const searchSourceJSON = get(doc, 'attributes.kibanaSavedObjectMeta.searchSourceJSON');
+  if (typeof searchSourceJSON !== 'string') {
+    return;
+  }
+  let searchSource;
+  try {
+    searchSource = JSON.parse(searchSourceJSON);
+  } catch (e) {
+    // Let it go, the data is invalid and we'll leave it as is
+    return;
+  }
+  if (!searchSource.index) {
+    return;
+  }
+  doc.references.push({
+    name: 'kibanaSavedObjectMeta.searchSourceJSON.index',
+    type: 'index-pattern',
+    id: searchSource.index,
+  });
+  searchSource.indexRefName = 'kibanaSavedObjectMeta.searchSourceJSON.index';
+  delete searchSource.index;
+  doc.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify(searchSource);
+}
+
 export const migrations = {
   visualization: {
     '7.0.0': (doc) => {
+      // Set new "references" attribute
+      doc.references = doc.references || [];
+
+      // Migrate index pattern
+      migrateIndexPattern(doc);
+
+      // Migrate saved search
+      const savedSearchId = get(doc, 'attributes.savedSearchId');
+      if (savedSearchId) {
+        doc.references.push({
+          type: 'search',
+          name: 'search_0',
+          id: savedSearchId,
+        });
+        doc.attributes.savedSearchRefName = 'search_0';
+        delete doc.attributes.savedSearchId;
+      }
+
+      // Migrate table splits
       try {
         const visState = JSON.parse(doc.attributes.visState);
         if (get(visState, 'type') !== 'table') {
@@ -55,5 +99,52 @@ export const migrations = {
         throw new Error(`Failure attempting to migrate saved object '${doc.attributes.title}' - ${e}`);
       }
     }
-  }
+  },
+  dashboard: {
+    '7.0.0': (doc) => {
+      // Set new "references" attribute
+      doc.references = doc.references || [];
+      // Migrate index pattern
+      migrateIndexPattern(doc);
+      // Migrate panels
+      const panelsJSON = get(doc, 'attributes.panelsJSON');
+      if (typeof panelsJSON !== 'string') {
+        return doc;
+      }
+      let panels;
+      try {
+        panels = JSON.parse(panelsJSON);
+      } catch (e) {
+        // Let it go, the data is invalid and we'll leave it as is
+        return doc;
+      }
+      if (!Array.isArray(panels)) {
+        return doc;
+      }
+      panels.forEach((panel, i) => {
+        if (!panel.type || !panel.id) {
+          return;
+        }
+        panel.panelRefName = `panel_${i}`;
+        doc.references.push({
+          name: `panel_${i}`,
+          type: panel.type,
+          id: panel.id,
+        });
+        delete panel.type;
+        delete panel.id;
+      });
+      doc.attributes.panelsJSON = JSON.stringify(panels);
+      return doc;
+    },
+  },
+  search: {
+    '7.0.0': (doc) => {
+      // Set new "references" attribute
+      doc.references = doc.references || [];
+      // Migrate index pattern
+      migrateIndexPattern(doc);
+      return doc;
+    },
+  },
 };