Skip to content

Commit

Permalink
[Rename] Refactored src/core/server/saved_objects directory (#91)
Browse files Browse the repository at this point in the history
* [Rename] Refactored src/core/server/saved_objects directory

Signed-off-by: Mihir Soni <mihirsoni.123@gmail.com>

* [Rename] Refactored src/core/server/saved_objects directory

Signed-off-by: Mihir Soni <mihirsoni.123@gmail.com>

* [Rename] addressed review feedbacks

* [Rename] addressed review feedbacks

* [Rename] addressed PR comments

Signed-off-by: Mihir Soni <mihirsoni.123@gmail.com>
  • Loading branch information
mihirsoni authored and kavilla committed Mar 19, 2021
1 parent 4a6a71a commit 274eac5
Show file tree
Hide file tree
Showing 93 changed files with 1,038 additions and 982 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import { mockUuidv4 } from './__mocks__';
import { savedObjectsClientMock } from '../../mocks';
import { SavedObjectReference, SavedObjectsImportRetry } from 'kibana/public';
import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public';
import { SavedObjectsClientContract, SavedObject } from '../types';
import { SavedObjectsErrorHelpers } from '..';
import { checkConflicts } from './check_conflicts';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ describe('getNonExistingReferenceAsKeys()', () => {
expect(savedObjectsClient.bulkGet).toHaveBeenCalledTimes(0);
});

test('removes references that exist within es', async () => {
test('removes references that exist within opensearch', async () => {
const savedObjects = [
{
id: '2',
Expand Down Expand Up @@ -477,7 +477,7 @@ describe('validateReferences()', () => {
expect(savedObjectsClient.bulkGet).not.toHaveBeenCalled();
});

test(`doesn't return errors when references exist in Elasticsearch`, async () => {
test(`doesn't return errors when references exist in OpenSearch`, async () => {
savedObjectsClient.bulkGet.mockResolvedValue({
saved_objects: [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,19 @@
import { IndexMapping } from '../types';

/**
* Get the property mappings for the root type in the EsMappingsDsl
* Get the property mappings for the root type in the OpenSearchMappingsDsl
*
* If the mappings don't have a root type, or the root type is not
* an object type (it's a keyword or something) this function will
* throw an error.
*
* EsPropertyMappings objects have the root property names as their
* OpenSearchPropertyMappings objects have the root property names as their
* first level keys which map to the mappings object for each property.
* If the property is of type object it too could have a `properties`
* key whose value follows the same format.
*
* This data can be found at `{indexName}.mappings.{typeName}.properties`
* in the es indices.get() response.
* in the opensearch indices.get() response.
*/
export function getRootProperties(mapping: IndexMapping) {
if (!mapping.properties) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,18 @@ import {
import { getRootProperties } from './get_root_properties';

/**
* Get the property mappings for the root type in the EsMappingsDsl
* Get the property mappings for the root type in the OpenSearchMappingsDsl
* where the properties are objects
*
* If the mappings don't have a root type, or the root type is not
* an object type (it's a keyword or something) this function will
* throw an error.
*
* This data can be found at `{indexName}.mappings.{typeName}.properties`
* in the es indices.get() response where the properties are objects.
* in the opensearch indices.get() response where the properties are objects.
*
* @param {EsMappingsDsl} mappings
* @return {EsPropertyMappings}
* @param {OpenSearchMappingsDsl} mappings
* @return {OpenSearchPropertyMappings}
*/

const omittedRootProps = ['migrationVersion', 'references'];
Expand Down
2 changes: 1 addition & 1 deletion src/core/server/saved_objects/mappings/lib/get_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import { IndexMapping } from '../types';

/**
* Get the names of the types defined in the EsMappingsDsl
* Get the names of the types defined in the OpenSearchMappingsDsl
*/
export function getTypes(mappings: IndexMapping) {
return Object.keys(mappings).filter((type) => type !== '_default_');
Expand Down
72 changes: 36 additions & 36 deletions src/core/server/saved_objects/migrations/README.md
Original file line number Diff line number Diff line change
@@ -1,48 +1,48 @@
# Saved Object Migrations

Migrations are the mechanism by which saved object indices are kept up to date with the Kibana system. Plugin authors write their plugins to work with a certain set of mappings, and documents of a certain shape. Migrations ensure that the index actually conforms to those expectations.
Migrations are the mechanism by which saved object indices are kept up to date with the OpenSearch Dashboards system. Plugin authors write their plugins to work with a certain set of mappings, and documents of a certain shape. Migrations ensure that the index actually conforms to those expectations.

## Migrating the index

When Kibana boots, prior to serving any requests, it performs a check to see if the kibana index needs to be migrated.
When OpenSearch Dashboards boots, prior to serving any requests, it performs a check to see if the opensearch-dashboards index needs to be migrated.

- If there are out of date docs, or mapping changes, or the current index is not aliased, the index is migrated.
- If the Kibana index does not exist, it is created.
- If the OpenSearch Dashboards index does not exist, it is created.

All of this happens prior to Kibana serving any http requests.
All of this happens prior to OpenSearch Dashboards serving any http requests.

Here is the gist of what happens if an index migration is necessary:

* If `.kibana` (or whatever the Kibana index is named) is not an alias, it will be converted to one:
* Reindex `.kibana` into `.kibana_1`
* Delete `.kibana`
* Create an alias `.kibana` that points to `.kibana_1`
* Create a `.kibana_2` index
* Copy all documents from `.kibana_1` into `.kibana_2`, running them through any applicable migrations
* Point the `.kibana` alias to `.kibana_2`
* If `.opensearch-dashboards` (or whatever the OpenSearch Dashboards index is named) is not an alias, it will be converted to one:
* Reindex `.opensearch-dashboards` into `.opensearch-dashboards_1`
* Delete `.opensearch-dashboards`
* Create an alias `.opensearch-dashboards` that points to `.opensearch-dashboards_1`
* Create a `.opensearch-dashboards_2` index
* Copy all documents from `.opensearch-dashboards_1` into `.opensearch-dashboards_2`, running them through any applicable migrations
* Point the `.opensearch-dashboards` alias to `.opensearch-dashboards_2`

## Migrating Kibana clusters
## Migrating OpenSearch Dashboards clusters

If Kibana is being run in a cluster, migrations will be coordinated so that they only run on one Kibana instance at a time. This is done in a fairly rudimentary way. Let's say we have two Kibana instances, kibana1 and kibana2.
If OpenSearch Dashboards is being run in a cluster, migrations will be coordinated so that they only run on one OpenSearch Dashboards instance at a time. This is done in a fairly rudimentary way. Let's say we have two OpenSearch Dashboards instances, opensearch-dashboards-1 and opensearch-dashboards-2.

* kibana1 and kibana2 both start simultaneously and detect that the index requires migration
* kibana1 begins the migration and creates index `.kibana_4`
* kibana2 tries to begin the migration, but fails with the error `.kibana_4 already exists`
* kibana2 logs that it failed to create the migration index, and instead begins polling
* Every few seconds, kibana2 instance checks the `.kibana` index to see if it is done migrating
* Once `.kibana` is determined to be up to date, the kibana2 instance continues booting
* opensearch-dashboards-1 and opensearch-dashboards-2 both start simultaneously and detect that the index requires migration
* opensearch-dashboards-1 begins the migration and creates index `.opensearch-dashboards_4`
* opensearch-dashboards-2 tries to begin the migration, but fails with the error `.opensearch-dashboards_4 already exists`
* opensearch-dashboards-2 logs that it failed to create the migration index, and instead begins polling
* Every few seconds, opensearch-dashboards-2 instance checks the `.opensearch-dashboards` index to see if it is done migrating
* Once `.opensearch-dashboards` is determined to be up to date, the opensearch-dashboards-2 instance continues booting

In this example, if the `.kibana_4` index existed prior to Kibana booting, the entire migration process will fail, as all Kibana instances will assume another instance is migrating to the `.kibana_4` index. This problem is only fixable by deleting the `.kibana_4` index.
In this example, if the `.opensearch-dashboards_4` index existed prior to OpenSearch Dashboards booting, the entire migration process will fail, as all OpenSearch Dashboards instances will assume another instance is migrating to the `.opensearch-dashboards_4` index. This problem is only fixable by deleting the `.opensearch-dashboards_4` index.

## Import / export

If a user attempts to import FanciPlugin 1.0 documents into a Kibana system that is running FanciPlugin 2.0, those documents will be migrated prior to being persisted in the Kibana index. If a user attempts to import documents having a migration version that is _greater_ than the current Kibana version, the documents will fail to import.
If a user attempts to import FanciPlugin 1.0 documents into a OpenSearch Dashboards system that is running FanciPlugin 2.0, those documents will be migrated prior to being persisted in the OpenSearch Dashboards index. If a user attempts to import documents having a migration version that is _greater_ than the current OpenSearch Dashboards version, the documents will fail to import.

## Validation

It might happen that a user modifies their FanciPlugin 1.0 export file to have documents with a migrationVersion of 2.0.0. In this scenario, Kibana will store those documents as if they are up to date, even though they are not, and the result will be unknown, but probably undesirable behavior.
It might happen that a user modifies their FanciPlugin 1.0 export file to have documents with a migrationVersion of 2.0.0. In this scenario, OpenSearch Dashboards will store those documents as if they are up to date, even though they are not, and the result will be unknown, but probably undesirable behavior.

Similarly, Kibana server APIs assume that they are sent up to date documents unless a document specifies a migrationVersion. This means that out-of-date callers of our APIs will send us out-of-date documents, and those documents will be accepted and stored as if they are up-to-date.
Similarly, OpenSearch Dashboards server APIs assume that they are sent up to date documents unless a document specifies a migrationVersion. This means that out-of-date callers of our APIs will send us out-of-date documents, and those documents will be accepted and stored as if they are up-to-date.

To prevent this from happening, migration authors should _always_ write a [validation](../validation) function that throws an error if a document is not up to date, and this validation function should always be updated any time a new migration is added for the relevent document types.

Expand All @@ -60,7 +60,7 @@ So, let's say we have a document that looks like this:
}
```

In this document, one plugin might own the `dashboard` type, and another plugin might own the `securityKey` type. If two or more plugins define securityKey migrations `{ migrations: { securityKey: { ... } } }`, Kibana will fail to start.
In this document, one plugin might own the `dashboard` type, and another plugin might own the `securityKey` type. If two or more plugins define securityKey migrations `{ migrations: { securityKey: { ... } } }`, OpenSearch Dashboards will fail to start.

To write a migration for this document, the dashboard plugin might look something like this:

Expand All @@ -86,31 +86,31 @@ uiExports: {
}
```

After Kibana migrates the index, our example document would have `{ attributes: { title: 'WHATEVER!!' } }`.
After OpenSearch Dashboards migrates the index, our example document would have `{ attributes: { title: 'WHATEVER!!' } }`.

Each migration function only needs to be able to handle documents belonging to the previous version. The initial migration function (in this example, `1.9.0`) needs to be more flexible, as it may be passed documents of any pre `1.9.0` shape.

## Disabled plugins

If a plugin is disbled, all of its documents are retained in the Kibana index. They can be imported and exported. When the plugin is re-enabled, Kibana will migrate any out of date documents that were imported or retained while it was disabled.
If a plugin is disbled, all of its documents are retained in the OpenSearch Dashboards index. They can be imported and exported. When the plugin is re-enabled, OpenSearch Dashboards will migrate any out of date documents that were imported or retained while it was disabled.

## Configuration

Kibana index migrations expose a few config settings which might be tweaked:
OpenSearch Dashboards index migrations expose a few config settings which might be tweaked:

* `migrations.scrollDuration` - The
[scroll](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#scroll-search-context)
value used to read batches of documents from the source index. Defaults to
`15m`.
* `migrations.batchSize` - The number of documents to read / transform / write
at a time during index migrations
* `migrations.pollInterval` - How often, in milliseconds, secondary Kibana
instances will poll to see if the primary Kibana instance has finished
* `migrations.pollInterval` - How often, in milliseconds, secondary OpenSearchDashboards
instances will poll to see if the primary OpenSearch Dashboards instance has finished
migrating the index.
* `migrations.skip` - Skip running migrations on startup (defaults to false).
This should only be used for running integration tests without a running
elasticsearch cluster. Note: even though migrations won't run on startup,
individual docs will still be migrated when read from ES.
opensearch cluster. Note: even though migrations won't run on startup,
individual docs will still be migrated when read from OpenSearch.

## Example

Expand Down Expand Up @@ -170,9 +170,9 @@ uiExports: {
}
```

Now, whenever Kibana boots, if FanciPlugin is enabled, Kibana scans its index for any documents that have type 'fanci' and have a `migrationVersion.fanci` property that is anything other than `2.0.0`. If any such documents are found, the index is determined to be out of date (or at least of the wrong version), and Kibana attempts to migrate the index.
Now, whenever OpenSearch Dashboards boots, if FanciPlugin is enabled, OpenSearch Dashboards scans its index for any documents that have type 'fanci' and have a `migrationVersion.fanci` property that is anything other than `2.0.0`. If any such documents are found, the index is determined to be out of date (or at least of the wrong version), and OpenSearch Dashboards attempts to migrate the index.

At the end of the migration, Kibana's fanci documents will look something like this:
At the end of the migration, OpenSearchDashboards's fanci documents will look something like this:

```js
{
Expand All @@ -191,10 +191,10 @@ Note, the migrationVersion property has been added, and it contains information

The migrations source code is grouped into two folders:

* `core` - Contains index-agnostic, general migration logic, which could be reused for indices other than `.kibana`
* `kibana` - Contains a relatively light-weight wrapper around core, which provides `.kibana` index-specific logic
* `core` - Contains index-agnostic, general migration logic, which could be reused for indices other than `.opensearch-dashboards`
* `opensearch-dashboards` - Contains a relatively light-weight wrapper around core, which provides `.opensearch-dashboards` index-specific logic

Generally, the code eschews classes in favor of functions and basic data structures. The publicly exported code is all class-based, however, in an attempt to conform to Kibana norms.
Generally, the code eschews classes in favor of functions and basic data structures. The publicly exported code is all class-based, however, in an attempt to conform to OpenSearch Dashboards norms.

### Core

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export function buildActiveMappings(
/**
* Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because
* actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes
* match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana
* match our expectations, we don't require a migration. This allows OpenSearch to tack on additional mappings that OpenSearchDashboards
* doesn't know about or expect, without triggering continual migrations.
*/
export function diffMappings(actual: IndexMapping, expected: IndexMapping) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ const createRegistry = (...types: Array<Partial<SavedObjectsType>>) => {

test('mappings without index pattern goes to default index', () => {
const result = createIndexMap({
kibanaIndexName: '.kibana',
opensearchDashboardsIndexName: '.opensearch-dashboards',
registry: createRegistry({
name: 'type1',
namespaceType: 'single',
Expand All @@ -54,7 +54,7 @@ test('mappings without index pattern goes to default index', () => {
},
});
expect(result).toEqual({
'.kibana': {
'.opensearch-dashboards': {
typeMappings: {
type1: {
properties: {
Expand All @@ -70,11 +70,11 @@ test('mappings without index pattern goes to default index', () => {

test(`mappings with custom index pattern doesn't go to default index`, () => {
const result = createIndexMap({
kibanaIndexName: '.kibana',
opensearchDashboardsIndexName: '.opensearch-dashboards',
registry: createRegistry({
name: 'type1',
namespaceType: 'single',
indexPattern: '.other_kibana',
indexPattern: '.other_opensearch_dashboards',
}),
indexMap: {
type1: {
Expand All @@ -87,7 +87,7 @@ test(`mappings with custom index pattern doesn't go to default index`, () => {
},
});
expect(result).toEqual({
'.other_kibana': {
'.other_opensearch_dashboards': {
typeMappings: {
type1: {
properties: {
Expand All @@ -103,11 +103,11 @@ test(`mappings with custom index pattern doesn't go to default index`, () => {

test('creating a script gets added to the index pattern', () => {
const result = createIndexMap({
kibanaIndexName: '.kibana',
opensearchDashboardsIndexName: '.opensearch-dashboards',
registry: createRegistry({
name: 'type1',
namespaceType: 'single',
indexPattern: '.other_kibana',
indexPattern: '.other_opensearch_dashboards',
convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
}),
indexMap: {
Expand All @@ -121,7 +121,7 @@ test('creating a script gets added to the index pattern', () => {
},
});
expect(result).toEqual({
'.other_kibana': {
'.other_opensearch_dashboards': {
script: `ctx._id = ctx._source.type + ':' + ctx._id`,
typeMappings: {
type1: {
Expand All @@ -137,7 +137,7 @@ test('creating a script gets added to the index pattern', () => {
});

test('throws when two scripts are defined for an index pattern', () => {
const defaultIndex = '.kibana';
const defaultIndex = '.opensearch-dashboards';
const registry = createRegistry(
{
name: 'type1',
Expand Down Expand Up @@ -169,11 +169,11 @@ test('throws when two scripts are defined for an index pattern', () => {
};
expect(() =>
createIndexMap({
kibanaIndexName: defaultIndex,
opensearchDashboardsIndexName: defaultIndex,
registry,
indexMap,
})
).toThrowErrorMatchingInlineSnapshot(
`"convertToAliasScript has been defined more than once for index pattern \\".kibana\\""`
`"convertToAliasScript has been defined more than once for index pattern \\".opensearch-dashboards\\""`
);
});
Loading

0 comments on commit 274eac5

Please sign in to comment.