[saved objects] enable deletion of saved objects by type if configured (

#6443)

* [saved objects] enable deletion of saved objects by type if configured

    Adds the following settings:
    ```
    migrations.delete.enabled
    migrations.delete.types
    ```

    `unknown` types already exist but the purpose of this type is for plugins
    that are disabled. OpenSearch Dashboards gets confused when a plugin is not
    defining a saved object type but the saved object exists. This can occur
    when migrating from a non-OSD version and there exists non-compatiable
    saved objects.

    If OSD is failing to migrate an index because of a document, I can now
    configure OSD to delete types of saved objects that I specified because
    I know that these types should not be carried over.

    resolves: #1040

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

    * address comments

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

    * Changeset file for PR #6443 created/updated

    ---------

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
	(cherry picked from commit 7a8c281)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

changelogs/fragments/6443.yml

@@ -0,0 +1,2 @@
+feat:
+- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443))

config/opensearch_dashboards.yml

@@ -317,6 +317,12 @@
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
 
+# Optional settings to specify saved object types to be deleted during migration.
+# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications.
+# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution.
+# migrations.delete.enabled: false
+# migrations.delete.types: []
+
 # Set the value to true to enable Ui Metric Collectors in Usage Collector
 # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
-# usageCollection.uiMetric.enabled: false
+# usageCollection.uiMetric.enabled: false

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -435,6 +435,228 @@ describe('IndexMigrator', () => {
  });
  });
 
+ test('deletes saved objects by type if configured', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ delete_type: { properties: { type: deleteType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
+ test('retains saved objects by type if delete is not enabled', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return false;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ delete_type: { properties: { type: deleteType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ delete_type: { dynamic: false, properties: {} },
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
+ test('retains saved objects by type if delete types does not exist', async () => {
+ const { client } = testOpts;
+
+ const deleteType = 'delete_type';
+ const retainType = 'retain_type';
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [deleteType];
+ }
+ });
+ testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+ testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+ withIndex(client, {
+ index: {
+ '.kibana_1': {
+ aliases: {},
+ mappings: {
+ properties: {
+ retain_type: { properties: { type: retainType } },
+ },
+ },
+ },
+ },
+ });
+
+ await new IndexMigrator(testOpts).migrate();
+
+ expect(client.indices.create).toHaveBeenCalledWith({
+ body: {
+ mappings: {
+ dynamic: 'strict',
+ _meta: {
+ migrationMappingPropertyHashes: {
+ foo: '625b32086eb1d1203564cf85062dd22e',
+ migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+ namespace: '2f4316de49999235636386fe51dc06c1',
+ namespaces: '2f4316de49999235636386fe51dc06c1',
+ originId: '2f4316de49999235636386fe51dc06c1',
+ references: '7997cf5a56cc02bdc9c93361bde732b0',
+ type: '2f4316de49999235636386fe51dc06c1',
+ updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+ },
+ },
+ properties: {
+ retain_type: { dynamic: false, properties: {} },
+ foo: { type: 'text' },
+ migrationVersion: { dynamic: 'true', type: 'object' },
+ namespace: { type: 'keyword' },
+ namespaces: { type: 'keyword' },
+ originId: { type: 'keyword' },
+ type: { type: 'keyword' },
+ updated_at: { type: 'date' },
+ references: {
+ type: 'nested',
+ properties: {
+ name: { type: 'keyword' },
+ type: { type: 'keyword' },
+ id: { type: 'keyword' },
+ },
+ },
+ },
+ },
+ settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+ },
+ index: '.kibana_2',
+ });
+ });
+
  test('points the alias at the dest index', async () => {
  const { client } = testOpts;
 

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -28,6 +28,7 @@
  * under the License.
  */
 
+import { DeleteByQueryRequest } from '@opensearch-project/opensearch/api/types';
 import { diffMappings } from './build_active_mappings';
 import * as Index from './opensearch_index';
 import { migrateRawDocs } from './migrate_raw_docs';
@@ -123,6 +124,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
  const { client, alias, source, dest, log } = context;
 
  await deleteIndexTemplates(context);
+ await deleteSavedObjectsByType(context);
 
  log.info(`Creating index ${dest.indexName}.`);
 
@@ -171,6 +173,33 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern
  return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! })));
 }
 
+/**
+ * Delete saved objects by type. If migrations.delete.types is specified,
+ * any saved objects that matches that type will be deleted.
+ */
+async function deleteSavedObjectsByType(context: Context) {
+ const { client, source, log, typesToDelete } = context;
+ if (!source.exists || !typesToDelete || typesToDelete.length === 0) {
+ return;
+ }
+
+ log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`);
+ const params = {
+ index: source.indexName,
+ body: {
+ query: {
+ bool: {
+ should: [...typesToDelete.map((type) => ({ term: { type } }))],
+ },
+ },
+ },
+ conflicts: 'proceed',
+ refresh: true,
+ } as DeleteByQueryRequest;
+ log.debug(`Delete by query params: ${JSON.stringify(params)}`);
+ return client.deleteByQuery(params);
+}
+
 /**
  * Moves all docs from sourceIndex to destIndex, migrating each as necessary.
  * This moves documents from the concrete index, rather than the alias, to prevent

src/core/server/saved_objects/migrations/core/migration_context.test.ts

@@ -28,7 +28,8 @@
  * under the License.
  */
 
-import { disableUnknownTypeMappingFields } from './migration_context';
+import { disableUnknownTypeMappingFields, deleteTypeMappingsFields } from './migration_context';
+import { configMock } from '../../../config/mocks';
 
 describe('disableUnknownTypeMappingFields', () => {
  const sourceMappings = {
@@ -97,3 +98,87 @@ describe('disableUnknownTypeMappingFields', () => {
  });
  });
 });
+
+describe('deleteTypeMappingsFields', () => {
+ it('should delete specified type mappings fields', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return ['type1', 'type3'];
+ }
+ });
+
+ const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(updatedMappings.properties).toEqual({
+ type2: { type: 'keyword' },
+ });
+ });
+
+ it('should not delete any type mappings fields if delete is not enabled', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return false;
+ }
+ if (path === 'migrations.delete.types') {
+ return ['type1', 'type3'];
+ }
+ });
+
+ const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(updatedMappings.properties).toEqual({
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ });
+ });
+
+ it('should not delete any type mappings fields if delete types are not specified', () => {
+ const targetMappings = {
+ properties: {
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ },
+ } as const;
+
+ const rawConfig = configMock.create();
+ rawConfig.get.mockImplementation((path) => {
+ if (path === 'migrations.delete.enabled') {
+ return true;
+ }
+ if (path === 'migrations.delete.types') {
+ return [];
+ }
+ });
+
+ const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);
+
+ expect(updatedMappings.properties).toEqual({
+ type1: { type: 'text' },
+ type2: { type: 'keyword' },
+ type3: { type: 'boolean' },
+ });
+ });
+});