From a6fc33ff544bfe397fa302c4483a6328e97336e0 Mon Sep 17 00:00:00 2001 From: Josh Dover Date: Thu, 6 Aug 2020 11:46:48 -0600 Subject: [PATCH] Add docs for SO migrations --- .../development-plugin-saved-objects.asciidoc | 226 +++++++++++++++++- 1 file changed, 224 insertions(+), 2 deletions(-) diff --git a/docs/developer/architecture/development-plugin-saved-objects.asciidoc b/docs/developer/architecture/development-plugin-saved-objects.asciidoc index ff1fdb6204985..5cd6aab9e81fa 100644 --- a/docs/developer/architecture/development-plugin-saved-objects.asciidoc +++ b/docs/developer/architecture/development-plugin-saved-objects.asciidoc @@ -25,7 +25,7 @@ included in the export. provide RBAC access control and the ability to organize Saved Objects into spaces. -This document contains developer guidelines and best-practises for plugins +This document contains developer guidelines and best-practices for plugins wanting to use Saved Objects. ==== Registering a Saved Object type @@ -202,4 +202,226 @@ up to date. If a visualization `id` was directly stored in `dashboard.panels[0].visualization` there is a risk that this `id` gets updated without updating the reference in the references array. -// TODO: Writing migrations \ No newline at end of file +==== Writing Migrations + +Saved Objects support schema changes between Kibana versions, which we call +migrations. Migrations are applied when a Kibana installation is upgraded from +one version to the next, when exports are imported via the Saved Objects +Management UI, or when a new object is created via the HTTP API. + +Each Saved Object type may define migrations for its schema. Migrations are +specified by the Kibana version number, receive an input document, and must +return the fully migrated document to be persisted to Elasticsearch. + +Let's say we want to define two migrations: +- In version 1.1.0, we want to drop the `subtitle` field and append it to the + title +- In version 1.4.0, we want to add a new `id` field to every panel with a newly + generated UUID. + +First, the current `mappings` should always reflect the latest or "target" +schema. Next, we should define a migration function for each step in the schema +evolution: + +src/plugins/my_plugin/server/saved_objects/dashboard_visualization.ts +[source,typescript] +---- +import { SavedObjectsType } from 'src/core/server'; + +export const dashboardVisualization: SavedObjectsType = { + name: 'dashboard_visualization', // <1> + /** ... */ + migrations: { + // Takes a pre 1.1.0 doc, and converts it to 1.1.0 + '1.1.0': (doc: DashboardVisualizationPre110): DashboardVisualization110 => { // <1> + return { + ...doc, // <2> + attributes: { + ...doc.attributes, + title: `${doc.attributes.title} - ${doc.attributes.subtitle}` + } + } + }, + + // Takes a 1.1.0 doc, and converts it to 1.4.0 + '1.4.0': (doc: DashboardVisualization110): DashboardVisualization140 => { // <3> + doc.attributes.panels = doc.attributes.panels.map(panel => { + panel.id = uuid.v4(); + return panel; + }); + return doc; + }, + }, +}; +---- +<1> It is useful to define an interface for each version of the schema. This +allows TypeScript to ensure that you are properly handling the input and output +types correctly as the schema evolves. +<2> Returning a shallow copy is necessary to avoid type errors when using +different types for the input and output shape. +<3> Migrations do not have to be defined for every version, only those in which +the schema needs to change. + +Migrations should be written defensively. If a document is encountered that is +not in the expected shape, migrations are encouraged to throw an exception to +abort the upgrade. + +===== Nested Migrations + +In some cases, objects may contain state that is functionally "owned" by other +plugins. An example is a dashboard that may contain state owned by specific +types of embeddables. In this case, the dashboard migrations should delegate +migrating this state to their functional owner, the individual embeddable types, +in order to compose a single migration function that handles all nested state. + +How the migration of the nested object is surfaced to the containing object is +up to plugin authors to define. In general, we encourage that registries that +expose state that may be persisted elsewhere include migrations as part of the +interface that is registered. This allows consumers of the registry items to +utilize these migrations on the nested pieces of state inside of the root +migration. + +To demonstrate this, let's imagine we have: +- A chart registry plugin which allows plugins to register multiple types of charts +- A bar chart plugin that implements and registers a type of chart +- A chart list plugin that contains a list of charts to display. This plugin +persists the state of the underlying charts and allows those charts to define +migrations for that state. + +The example code here is length, but necessary to demonstrate how this different +pieces fit together. + +src/plugins/dashboard/server/saved_objects/dashboard.ts +[source,typescript] +---- +type ChartMigrationFn = (chartDoc: object) => ChartState; + +interface Chart { + type: string; + render(chart: ChartState, element: HTMLElement): void; + create(): ChartState; + getMigrations(): Record>; +} + +class ChartRegistryPlugin { + private readonly charts = new Map(); + public setup() { + return { + register(chart: Chart) { + charts.set(chart.id, chart); + }, + + /** Returns migrations by version that can handle migrating a chart of any type */ + getMigrations() { + // Here we rollup the migrations from each individual chart implementation to create a single migration function + // for each version that will the chart state migration if the input chart is of the same type. + const chartMigrations = this.charts.reduce((migrations, chart) => { + for (const [version, chartMigration] of Object.entries(chart.getMigrations)) { + const existingMigration = migrations[version] ?? (input: any) => input; + migrations[version] = (chartDoc) => { + if (chartDoc.type === chart.type) { + chartDoc = chartMigration(chartDoc); + } + + return existingMigration(chartDoc); + } + } + }, {} as Record); + + return { + '1.1.0': (oldChart) => { + return { + ...oldChart, + // Apply the migrations on the chart state if and only if any charts define migrations for this version. + // It's important that the registry items can only access the `state` part of the complete chart object. + // This is the part of the document that the chart implementation functionally "owns". + state: chartMigrations['1.1.0'] ? chartMigrations['1.1.0'](oldChart.chartState) : oldChart.chartState + } + } + } + } + } + } +} + +interface BarChartState100 { + xAxis: string; + yAxis: string; + indexPattern: string; +} + +interface BarChartState110 { + xAxis: string; + yAxis: string; + dataSource: { + indexPattern: string; + } +} + +class BarChartPlugin { + public setup(core, plugins) { + plugins.charts.register({ + type: 'bar-chart', + render() { ... }, + create() { return { xAxis: 'foo', yAxis: 'bar', dataSource: { indexPattern: 'baz' } }}, + getMigrations() { + return { + '1.1.0': (oldChartState: BarChartState100): BarChartState110 => ({ + xAxis: oldChartState.xAxis, + yAxis: oldChartState.yAxis, + dataSource: { indexPattern: oldChartState.dataSource } + }) + } + } + }) + } +} + +class ChartListPlugin { + public setup(core, plugins) { + core.savedObjects.registerType({ + name: 'chart-list', + hidden: false, + mappings: { + dynamic: false, + properties: { + title: { + type: 'text', + }, + // We will store charts as an array, no need to index this field. + charts: { + index: false; + } + }, + }, + getMigrations() { + // Request the migrations for the chart state from the charts registry + const chartMigrations = plugins.charts.getMigrations(); + + return { + '1.1.0': (chartList) => { + return { + ...chartList, + // For each chart, apply the chart migration for this version if it exists + charts: chartList.map( + chart => chartMigrations['1.1.0'] ? chartMigrations['1.1.0'](chart) : chart + ) + } + } + } + } + }) + } +} +---- + +For this to all work, a few things must be true: +- Nested state that comes from pluggable sources must be isolated within the +containing object's schema. In this example, the state is isolated within the +chart's `state` field. +- Pluggable sources must provide their migrations to the registry. +- Containing objects must have a mechanism for locating these migrations. + +What about nested nested nested objects? It's turtles all the way down! These +migrations should all follow the same pattern to compose the complete migration +from migrations provided by their functional owners.