[saved objects] enable deletion of saved objects by type if configured (

#6443) (#6716)

* [saved objects] enable deletion of saved objects by type if configured

    Adds the following settings:
    ```
    migrations.delete.enabled
    migrations.delete.types
    ```

    `unknown` types already exist but the purpose of this type is for plugins
    that are disabled. OpenSearch Dashboards gets confused when a plugin is not
    defining a saved object type but the saved object exists. This can occur
    when migrating from a non-OSD version and there exists non-compatiable
    saved objects.

    If OSD is failing to migrate an index because of a document, I can now
    configure OSD to delete types of saved objects that I specified because
    I know that these types should not be carried over.

    resolves: #1040



    * address comments



    * Changeset file for PR #6443 created/updated

    ---------



	(cherry picked from commit 7a8c281)

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: Qingyang(Abby) Hu <abigailhu2000@gmail.com>
Co-authored-by: Kawika Avilla <kavilla414@gmail.com>
Co-authored-by: Qingyang(Abby) Hu <abigailhu2000@gmail.com>
Co-authored-by: ZilongX <99905560+ZilongX@users.noreply.github.com>

changelogs/fragments/6443.yml

@@ -0,0 +1,2 @@
+feat:
+- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443))

config/opensearch_dashboards.yml

@@ -322,6 +322,12 @@
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
 
+# Optional settings to specify saved object types to be deleted during migration.
+# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications.
+# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution.
+# migrations.delete.enabled: false
+# migrations.delete.types: []
+
 # Set the value to true to enable Ui Metric Collectors in Usage Collector
 # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
 # usageCollection.uiMetric.enabled: false

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -435,6 +435,228 @@ describe('IndexMigrator', () => {
     });
   });
 
+  test('deletes saved objects by type if configured', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              delete_type: { properties: { type: deleteType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
+  test('retains saved objects by type if delete is not enabled', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return false;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              delete_type: { properties: { type: deleteType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            delete_type: { dynamic: false, properties: {} },
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
+  test('retains saved objects by type if delete types does not exist', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+    const retainType = 'retain_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              retain_type: { properties: { type: retainType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            retain_type: { dynamic: false, properties: {} },
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
   test('points the alias at the dest index', async () => {
     const { client } = testOpts;
 

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -28,6 +28,7 @@
  * under the License.
  */
 
+import { DeleteByQueryRequest } from '@opensearch-project/opensearch/api/types';
 import { diffMappings } from './build_active_mappings';
 import * as Index from './opensearch_index';
 import { migrateRawDocs } from './migrate_raw_docs';
@@ -123,6 +124,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
   const { client, alias, source, dest, log } = context;
 
   await deleteIndexTemplates(context);
+  await deleteSavedObjectsByType(context);
 
   log.info(`Creating index ${dest.indexName}.`);
 
@@ -171,6 +173,33 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern
   return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! })));
 }
 
+/**
+ * Delete saved objects by type. If migrations.delete.types is specified,
+ * any saved objects that matches that type will be deleted.
+ */
+async function deleteSavedObjectsByType(context: Context) {
+  const { client, source, log, typesToDelete } = context;
+  if (!source.exists || !typesToDelete || typesToDelete.length === 0) {
+    return;
+  }
+
+  log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`);
+  const params = {
+    index: source.indexName,
+    body: {
+      query: {
+        bool: {
+          should: [...typesToDelete.map((type) => ({ term: { type } }))],
+        },
+      },
+    },
+    conflicts: 'proceed',
+    refresh: true,
+  } as DeleteByQueryRequest;
+  log.debug(`Delete by query params: ${JSON.stringify(params)}`);
+  return client.deleteByQuery(params);
+}
+
 /**
  * Moves all docs from sourceIndex to destIndex, migrating each as necessary.
  * This moves documents from the concrete index, rather than the alias, to prevent

src/core/server/saved_objects/migrations/core/migration_context.test.ts

@@ -28,7 +28,8 @@
  * under the License.
  */
 
-import { disableUnknownTypeMappingFields } from './migration_context';
+import { disableUnknownTypeMappingFields, deleteTypeMappingsFields } from './migration_context';
+import { configMock } from '../../../config/mocks';
 
 describe('disableUnknownTypeMappingFields', () => {
   const sourceMappings = {
@@ -97,3 +98,87 @@ describe('disableUnknownTypeMappingFields', () => {
     });
   });
 });
+
+describe('deleteTypeMappingsFields', () => {
+  it('should delete specified type mappings fields', () => {
+    const targetMappings = {
+      properties: {
+        type1: { type: 'text' },
+        type2: { type: 'keyword' },
+        type3: { type: 'boolean' },
+      },
+    } as const;
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return ['type1', 'type3'];
+      }
+    });
+
+    const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+    expect(updatedMappings.properties).toEqual({
+      type2: { type: 'keyword' },
+    });
+  });
+
+  it('should not delete any type mappings fields if delete is not enabled', () => {
+    const targetMappings = {
+      properties: {
+        type1: { type: 'text' },
+        type2: { type: 'keyword' },
+        type3: { type: 'boolean' },
+      },
+    } as const;
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return false;
+      }
+      if (path === 'migrations.delete.types') {
+        return ['type1', 'type3'];
+      }
+    });
+
+    const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+    expect(updatedMappings.properties).toEqual({
+      type1: { type: 'text' },
+      type2: { type: 'keyword' },
+      type3: { type: 'boolean' },
+    });
+  });
+
+  it('should not delete any type mappings fields if delete types are not specified', () => {
+    const targetMappings = {
+      properties: {
+        type1: { type: 'text' },
+        type2: { type: 'keyword' },
+        type3: { type: 'boolean' },
+      },
+    } as const;
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return [];
+      }
+    });
+
+    const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+    expect(updatedMappings.properties).toEqual({
+      type1: { type: 'text' },
+      type2: { type: 'keyword' },
+      type3: { type: 'boolean' },
+    });
+  });
+});