[saved objects] enable deletion of saved objects by type if configured

Adds the following settings:
```
migrations.delete.enabled
migrations.delete.types
```

`unknown` types already exist but the purpose of this type is for plugins
that are disabled. OpenSearch Dashboards gets confused when a plugin is not
defining a saved object type but the saved object exists. This can occur
when migrating from a non-OSD version and there exists non-compatiable
saved objects.

If OSD is failing to migrate an index because of a document, I can now
configure OSD to delete types of saved objects that I specified because
I know that these types should not be carried over.

resolves: opensearch-project#1040

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

CHANGELOG.md

@@ -91,6 +91,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Dynamic Configurations] Improve dynamic configurations by adding cache and simplifying client fetch ([#6364](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6364))
 - [MD] Add OpenSearch cluster group label to top of single selectable dropdown ([#6400](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6400))
 - [Workspace] Support workspace in saved objects client in server side. ([#6365](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6365))
+- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443))
 
 ### 🐛 Bug Fixes
 

config/opensearch_dashboards.yml

@@ -313,3 +313,9 @@
 
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
+
+# Optional settings to specify saved object types to be deleted during migration.
+# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications.
+# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution.
+# migrations.delete.enabled: false
+# migrations.delete.types: []

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -435,6 +435,228 @@ describe('IndexMigrator', () => {
  });
  });
 
+ test('deletes saved objects by type if configured', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ delete_type: { properties: { type: deleteType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
+ test('retains saved objects by type if delete is not enabled', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return false;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ delete_type: { properties: { type: deleteType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ delete_type: { dynamic: false, properties: {} },
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
+ test('retains saved objects by type if delete types does not exist', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+ const retainType = 'retain_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ retain_type: { properties: { type: retainType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ retain_type: { dynamic: false, properties: {} },
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
  test('points the alias at the dest index', async () => {
  const { client } = testOpts;
 

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -123,6 +123,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
  const { client, alias, source, dest, log } = context;
 
  await deleteIndexTemplates(context);
+ await deleteSavedObjectsByType(context);
 
  log.info(`Creating index ${dest.indexName}.`);
 
@@ -171,6 +172,31 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern
  return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! })));
 }
 
+/**
+ * Delete saved objects by type. This is used to remove saved object types that
+ * are not compatible with the current version of OpenSearch Dashboards.
+ */
+async function deleteSavedObjectsByType(context: Context) {
+ const { client, source, log, typesToDelete } = context;
+ if (!source.exists || !typesToDelete || typesToDelete.length === 0) {
+ return;
+ }
+
+ log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`);
+ return client.deleteByQuery({
+ index: source.indexName,
+ body: {
+ query: {
+ bool: {
+ should: [...typesToDelete.map((type) => ({ term: { type } }))],
+ },
+ },
+ },
+ conflicts: 'proceed',
+ refresh: true,
+ });
+}
+
 /**
  * Moves all docs from sourceIndex to destIndex, migrating each as necessary.
  * This moves documents from the concrete index, rather than the alias, to prevent

src/core/server/saved_objects/migrations/core/migration_context.test.ts

@@ -28,7 +28,8 @@
  * under the License.
  */
 
-import { disableUnknownTypeMappingFields } from './migration_context';
+import { disableUnknownTypeMappingFields, deleteTypeMappingsFields } from './migration_context';
+import { configMock } from '../../../config/mocks';
 
 describe('disableUnknownTypeMappingFields', () => {
  const sourceMappings = {
@@ -97,3 +98,87 @@ describe('disableUnknownTypeMappingFields', () => {
  });
  });
 });
+
+describe('deleteTypeMappingsFields', () => {
+ it('should delete specified type mappings fields', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return ['type1', 'type3'];
+ }
+ });
+
+ deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(targetMappings.properties).toEqual({
+ type2: { type: 'keyword' },
+ });
+ });
+
+ it('should not delete any type mappings fields if delete is not enabled', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return false;
+ }
+ if (path === 'migrations.delete.types') {
+ return ['type1', 'type3'];
+ }
+ });
+
+ deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(targetMappings.properties).toEqual({
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ });
+ });
+
+ it('should not delete any type mappings fields if delete types are not specified', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [];
+ }
+ });
+
+ deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(targetMappings.properties).toEqual({
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ });
+ });
+});