Import / Export API for Dashboards (#10858)

* Initial implementation

* Deduping final data array. Adding index patterns from saved searches

* Adding import

* Finishing import implimentation

* Adding tests for export

* Adding tests for import

* Filtering out bad ids

* Adding options for exclude and force

* Fixes per request by PR reviewers

* Adding a check for empty ids

* Use SavedObject API

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

* Omits missed objects, adds tests

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

* Moves import to SavedObjectsClient

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

* Fixing a bug with missing index patterns

* Fixing spacing issues (because the format is weird on this array which is too much for eslint to handle)

* Fixing tests and renaming file

* Changing paths to /api/kibana/dashboards/(export|import); adding validation to export

* Single call signature and use async/await

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

* Adding try/catch to JSON parses; Removing payload from export;

* removing redundent code

* Changing test to only use query arguments

* Removing bad panel from test data

* Return errors for bulkCreate objects

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>

* Changing everything to named imports; removing deps from everything; fixing tests to reflect named imports

* Refactoring to use async/await pattern

src/core_plugins/kibana/index.js

@@ -6,6 +6,8 @@ import { mkdirp as mkdirpNode } from 'mkdirp';
 import manageUuid from './server/lib/manage_uuid';
 import search from './server/routes/api/search';
 import settings from './server/routes/api/settings';
+import { importApi } from './server/routes/api/import';
+import { exportApi } from './server/routes/api/export';
 import scripts from './server/routes/api/scripts';
 import { registerSuggestionsApi } from './server/routes/api/suggestions';
 import * as systemApi from './server/lib/system_api';
@@ -126,6 +128,8 @@ export default function (kibana) {
       search(server);
       settings(server);
       scripts(server);
+      importApi(server);
+      exportApi(server);
       registerSuggestionsApi(server);
 
       server.expose('systemApi', systemApi);

src/core_plugins/kibana/server/lib/export/__tests__/collect_dashboards.js

@@ -0,0 +1,68 @@
+import sinon from 'sinon';
+import * as deps from '../collect_panels';
+import { collectDashboards } from '../collect_dashboards';
+import { expect } from 'chai';
+
+describe('collectDashboards(req, ids)', () => {
+
+  let collectPanelsStub;
+  const savedObjectsClient = { bulkGet: sinon.mock() };
+
+  const ids = ['dashboard-01', 'dashboard-02'];
+
+  beforeEach(() => {
+    collectPanelsStub = sinon.stub(deps, 'collectPanels');
+    collectPanelsStub.onFirstCall().returns(Promise.resolve([
+      { id: 'dashboard-01' },
+      { id: 'panel-01' },
+      { id: 'index-*' }
+    ]));
+    collectPanelsStub.onSecondCall().returns(Promise.resolve([
+      { id: 'dashboard-02' },
+      { id: 'panel-01' },
+      { id: 'index-*' }
+    ]));
+
+    savedObjectsClient.bulkGet.returns(Promise.resolve([
+      { id: 'dashboard-01' }, { id: 'dashboard-02' }
+    ]));
+  });
+
+  afterEach(() => {
+    collectPanelsStub.restore();
+    savedObjectsClient.bulkGet.reset();
+  });
+
+  it('should request all dashboards', async () => {
+    await collectDashboards(savedObjectsClient, ids);
+
+    expect(savedObjectsClient.bulkGet.calledOnce).to.equal(true);
+
+    const args = savedObjectsClient.bulkGet.getCall(0).args;
+    expect(args[0]).to.eql([{
+      id: 'dashboard-01',
+      type: 'dashboard'
+    }, {
+      id: 'dashboard-02',
+      type: 'dashboard'
+    }]);
+  });
+
+  it('should call collectPanels with dashboard docs', async () => {
+    await collectDashboards(savedObjectsClient, ids);
+
+    expect(collectPanelsStub.calledTwice).to.equal(true);
+    expect(collectPanelsStub.args[0][1]).to.eql({ id: 'dashboard-01' });
+    expect(collectPanelsStub.args[1][1]).to.eql({ id: 'dashboard-02' });
+  });
+
+  it('should return an unique list of objects', async () => {
+    const results = await collectDashboards(savedObjectsClient, ids);
+    expect(results).to.eql([
+      { id: 'dashboard-01' },
+      { id: 'panel-01' },
+      { id: 'index-*' },
+      { id: 'dashboard-02' },
+    ]);
+  });
+});

src/core_plugins/kibana/server/lib/export/__tests__/collect_index_patterns.js

@@ -0,0 +1,85 @@
+import sinon from 'sinon';
+import { collectIndexPatterns } from '../collect_index_patterns';
+import { expect } from 'chai';
+
+describe('collectIndexPatterns(req, panels)', () => {
+  const panels = [
+    {
+      attributes: {
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: JSON.stringify({ index: 'index-*' })
+        }
+      }
+    }, {
+      attributes: {
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: JSON.stringify({ index: 'logstash-*' })
+        }
+      }
+    }, {
+      attributes: {
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: JSON.stringify({ index: 'logstash-*' })
+        }
+      }
+    }, {
+      attributes: {
+        savedSearchId: 1,
+        kibanaSavedObjectMeta: {
+          searchSourceJSON: JSON.stringify({ index: 'bad-*' })
+        }
+      }
+    }
+  ];
+
+  const savedObjectsClient = { bulkGet: sinon.mock() };
+
+  beforeEach(() => {
+    savedObjectsClient.bulkGet.returns(Promise.resolve([
+      { id: 'index-*' }, { id: 'logstash-*' }
+    ]));
+  });
+
+  afterEach(() => {
+    savedObjectsClient.bulkGet.reset();
+  });
+
+  it('should request all index patterns', async () => {
+    await collectIndexPatterns(savedObjectsClient, panels);
+
+    expect(savedObjectsClient.bulkGet.calledOnce).to.equal(true);
+    expect(savedObjectsClient.bulkGet.getCall(0).args[0]).to.eql([{
+      id: 'index-*',
+      type: 'index-pattern'
+    }, {
+      id: 'logstash-*',
+      type: 'index-pattern'
+    }]);
+  });
+
+  it('should return the index pattern docs', async () => {
+    const results = await collectIndexPatterns(savedObjectsClient, panels);
+
+    expect(results).to.eql([
+      { id: 'index-*' },
+      { id: 'logstash-*' }
+    ]);
+  });
+
+  it('should return an empty array if nothing is requested', async () => {
+    const input = [
+      {
+        attributes: {
+          savedSearchId: 1,
+          kibanaSavedObjectMeta: {
+            searchSourceJSON: JSON.stringify({ index: 'bad-*' })
+          }
+        }
+      }
+    ];
+
+    const results = await collectIndexPatterns(savedObjectsClient, input);
+    expect(results).to.eql([]);
+    expect(savedObjectsClient.bulkGet.calledOnce).to.eql(false);
+  });
+});

src/core_plugins/kibana/server/lib/export/__tests__/collect_panels.js

@@ -0,0 +1,84 @@
+import sinon from 'sinon';
+import * as collectIndexPatternsDep from '../collect_index_patterns';
+import * as collectSearchSourcesDep from '../collect_search_sources';
+import { collectPanels } from '../collect_panels';
+import { expect } from 'chai';
+
+describe('collectPanels(req, dashboard)', () => {
+  let collectSearchSourcesStub;
+  let collectIndexPatternsStub;
+  let dashboard;
+
+  const savedObjectsClient = { bulkGet: sinon.mock() };
+
+  beforeEach(() => {
+    dashboard = {
+      attributes: {
+        panelsJSON: JSON.stringify([
+          { id: 'panel-01', type: 'search' },
+          { id: 'panel-02', type: 'visualization' }
+        ])
+      }
+    };
+
+    savedObjectsClient.bulkGet.returns(Promise.resolve([
+      { id: 'panel-01' }, { id: 'panel-02' }
+    ]));
+
+    collectIndexPatternsStub = sinon.stub(collectIndexPatternsDep, 'collectIndexPatterns');
+    collectIndexPatternsStub.returns([{ id: 'logstash-*' }]);
+    collectSearchSourcesStub = sinon.stub(collectSearchSourcesDep, 'collectSearchSources');
+    collectSearchSourcesStub.returns([ { id: 'search-01' }]);
+  });
+
+  afterEach(() => {
+    collectSearchSourcesStub.restore();
+    collectIndexPatternsStub.restore();
+    savedObjectsClient.bulkGet.reset();
+  });
+
+  it('should request each panel in the panelJSON', async () => {
+    await collectPanels(savedObjectsClient, dashboard);
+
+    expect(savedObjectsClient.bulkGet.calledOnce).to.equal(true);
+    expect(savedObjectsClient.bulkGet.getCall(0).args[0]).to.eql([{
+      id: 'panel-01',
+      type: 'search'
+    }, {
+      id: 'panel-02',
+      type: 'visualization'
+    }]);
+  });
+
+  it('should call collectSearchSources()', async () => {
+    await collectPanels(savedObjectsClient, dashboard);
+    expect(collectSearchSourcesStub.calledOnce).to.equal(true);
+    expect(collectSearchSourcesStub.args[0][1]).to.eql([
+      { id: 'panel-01' },
+      { id: 'panel-02' }
+    ]);
+  });
+
+  it('should call collectIndexPatterns()', async () => {
+    await collectPanels(savedObjectsClient, dashboard);
+
+    expect(collectIndexPatternsStub.calledOnce).to.equal(true);
+    expect(collectIndexPatternsStub.args[0][1]).to.eql([
+      { id: 'panel-01' },
+      { id: 'panel-02' }
+    ]);
+  });
+
+  it('should return panels, index patterns, search sources, and dashboard', async () => {
+    const results = await collectPanels(savedObjectsClient, dashboard);
+
+    expect(results).to.eql([
+      { id: 'panel-01' },
+      { id: 'panel-02' },
+      { id: 'logstash-*' },
+      { id: 'search-01' },
+      dashboard
+    ]);
+  });
+
+});

src/core_plugins/kibana/server/lib/export/__tests__/collect_search_sources.js

@@ -0,0 +1,64 @@
+import sinon from 'sinon';
+import * as deps from '../collect_index_patterns';
+import { collectSearchSources } from '../collect_search_sources';
+import { expect } from 'chai';
+describe('collectSearchSources(req, panels)', () => {
+  const savedObjectsClient = { bulkGet: sinon.mock() };
+
+  let panels;
+  let collectIndexPatternsStub;
+
+  beforeEach(() => {
+    panels = [
+      { attributes: { savedSearchId: 1 } },
+      { attributes: { savedSearchId: 2 } }
+    ];
+
+    collectIndexPatternsStub = sinon.stub(deps, 'collectIndexPatterns');
+    collectIndexPatternsStub.returns(Promise.resolve([{ id: 'logstash-*' }]));
+
+    savedObjectsClient.bulkGet.returns(Promise.resolve([
+      { id: 1 }, { id: 2 }
+    ]));
+  });
+
+  afterEach(() => {
+    collectIndexPatternsStub.restore();
+    savedObjectsClient.bulkGet.reset();
+  });
+
+  it('should request all search sources', async () => {
+    await collectSearchSources(savedObjectsClient, panels);
+
+    expect(savedObjectsClient.bulkGet.calledOnce).to.equal(true);
+    expect(savedObjectsClient.bulkGet.getCall(0).args[0]).to.eql([
+      { type: 'search', id: 1 }, { type: 'search', id: 2 }
+    ]);
+  });
+
+  it('should return the search source and index patterns', async () => {
+    const results = await collectSearchSources(savedObjectsClient, panels);
+
+    expect(results).to.eql([
+      { id: 1 },
+      { id: 2 },
+      { id: 'logstash-*' }
+    ]);
+  });
+
+  it('should return an empty array if nothing is requested', async () => {
+    const input = [
+      {
+        attributes: {
+          kibanaSavedObjectMeta: {
+            searchSourceJSON: JSON.stringify({ index: 'bad-*' })
+          }
+        }
+      }
+    ];
+
+    const results = await collectSearchSources(savedObjectsClient, input);
+    expect(results).to.eql([]);
+    expect(savedObjectsClient.bulkGet.calledOnce).to.eql(false);
+  });
+});

src/core_plugins/kibana/server/lib/export/__tests__/export_dashboards.js

@@ -0,0 +1,52 @@
+import * as deps from '../collect_dashboards';
+import { exportDashboards } from '../export_dashboards';
+import sinon from 'sinon';
+import { expect } from 'chai';
+
+describe('exportDashboards(req)', () => {
+
+  let req;
+  let collectDashboardsStub;
+
+  beforeEach(() => {
+    req = {
+      query: { dashboard: 'dashboard-01' },
+      server: {
+        config: () => ({ get: () => '6.0.0' }),
+        plugins: {
+          elasticsearch: {
+            getCluster: () => ({ callWithRequest: sinon.stub() })
+          }
+        },
+      }
+    };
+
+    collectDashboardsStub = sinon.stub(deps, 'collectDashboards');
+    collectDashboardsStub.returns(Promise.resolve([
+      { id: 'dasboard-01' },
+      { id: 'logstash-*' },
+      { id: 'panel-01' }
+    ]));
+  });
+
+  afterEach(() => {
+    collectDashboardsStub.restore();
+  });
+
+  it('should return a response object with version', () => {
+    return exportDashboards(req).then((resp) => {
+      expect(resp).to.have.property('version', '6.0.0');
+    });
+  });
+
+  it('should return a response object with objects', () => {
+    return exportDashboards(req).then((resp) => {
+      expect(resp).to.have.property('objects');
+      expect(resp.objects).to.eql([
+        { id: 'dasboard-01' },
+        { id: 'logstash-*' },
+        { id: 'panel-01' }
+      ]);
+    });
+  });
+});