diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/startup_migrations/run_startup_migrations.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/startup_migrations/run_startup_migrations.ts new file mode 100644 index 0000000000000..04b5c1158e411 --- /dev/null +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/startup_migrations/run_startup_migrations.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { CoreSetup, Logger } from '@kbn/core/server'; +import { errors } from '@elastic/elasticsearch'; +import { LockManagerService, isLockAcquisitionError } from '@kbn/lock-manager'; +import { resourceNames } from '..'; +import type { ObservabilityAIAssistantPluginStartDependencies } from '../../types'; +import type { ObservabilityAIAssistantConfig } from '../../config'; +import { reIndexKnowledgeBaseWithLock } from '../knowledge_base_service/reindex_knowledge_base'; +import { hasKbWriteIndex } from '../knowledge_base_service/has_kb_index'; +import { updateExistingIndexAssets } from '../index_assets/update_existing_index_assets'; + +const PLUGIN_STARTUP_LOCK_ID = 'observability_ai_assistant:startup_migrations'; + +// This function performs necessary startup migrations for the observability AI assistant: +// 1. Updates index assets to ensure mappings are correct +// 2. If the knowledge base index does not support the `semantic_text` field, it is re-indexed. +// 3. Populates the `semantic_text` field for knowledge base entries +export async function runStartupMigrations({ + core, + logger, + config, +}: { + core: CoreSetup; + logger: Logger; + config: ObservabilityAIAssistantConfig; +}) { + // update index assets to ensure mappings are correct + await updateExistingIndexAssets({ logger, core }); + + const [coreStart] = await core.getStartServices(); + const esClient = coreStart.elasticsearch.client; + + const lmService = new LockManagerService(core, logger); + await lmService + .withLock(PLUGIN_STARTUP_LOCK_ID, async () => { + const doesKbIndexExist = await hasKbWriteIndex({ esClient }); + + if (!doesKbIndexExist) { + logger.info('Knowledge base index does not exist. Aborting updating index assets'); + return; + } + + const isKbSemanticTextCompatible = await isKnowledgeBaseSemanticTextCompatible({ + logger, + esClient, + }); + + if (!isKbSemanticTextCompatible) { + await reIndexKnowledgeBaseWithLock({ core, logger, esClient }); + } + }) + .catch((error) => { + // we should propogate the error if it is not a LockAcquisitionError + if (!isLockAcquisitionError(error)) { + throw error; + } + logger.info('Startup migrations are already in progress. Aborting startup migrations'); + }); +} + +// Checks if the knowledge base index supports `semantic_text` +// If the index was created before version 8.11, it requires re-indexing to support the `semantic_text` field. +async function isKnowledgeBaseSemanticTextCompatible({ + logger, + esClient, +}: { + logger: Logger; + esClient: { asInternalUser: ElasticsearchClient }; +}): Promise { + const indexSettingsResponse = await esClient.asInternalUser.indices.getSettings({ + index: resourceNames.writeIndexAlias.kb, + }); + + const results = Object.entries(indexSettingsResponse); + if (results.length === 0) { + logger.debug('No knowledge base indices found. Skipping re-indexing.'); + return true; + } + + const [indexName, { settings }] = results[0]; + const createdVersion = parseInt(settings?.index?.version?.created ?? '', 10); + + // Check if the index was created before version 8.11 + const versionThreshold = 8110000; // Version 8.11.0 + if (createdVersion >= versionThreshold) { + logger.debug( + `Knowledge base index "${indexName}" was created in version ${createdVersion}, and does not require re-indexing. Semantic text field is already supported. Aborting` + ); + return true; + } + + logger.info( + `Knowledge base index was created in ${createdVersion} and must be re-indexed in order to support semantic_text field. Re-indexing now...` + ); + + return false; +} + +export function isSemanticTextUnsupportedError(error: Error) { + const semanticTextUnsupportedError = + '[semantic_text] is available on indices created with 8.11 or higher. Please create a new index to use [semantic_text]'; + + const isSemanticTextUnspported = + error instanceof errors.ResponseError && + (error.message.includes(semanticTextUnsupportedError) || + // @ts-expect-error + error.meta?.body?.error?.caused_by?.reason.includes(semanticTextUnsupportedError)); + + return isSemanticTextUnspported; +} diff --git a/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts new file mode 100644 index 0000000000000..da63a79bce1f3 --- /dev/null +++ b/x-pack/solutions/observability/test/api_integration_deployment_agnostic/apis/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import * as semver from 'semver'; +import { InferenceModelState } from '@kbn/observability-ai-assistant-plugin/common'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { getKbIndexCreatedVersion } from '../utils/knowledge_base'; +import { + TINY_ELSER_INFERENCE_ID, + TINY_ELSER_MODEL_ID, + setupTinyElserModelAndInferenceEndpoint, + teardownTinyElserModelAndInferenceEndpoint, +} from '../utils/model_and_inference'; +import { + createOrUpdateIndexAssets, + deleteIndexAssets, + restoreIndexAssets, + runStartupMigrations, +} from '../utils/index_assets'; +import { restoreKbSnapshot } from '../utils/snapshots'; + +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi'); + const es = getService('es'); + const retry = getService('retry'); + const log = getService('log'); + + // Sparse vector field was introduced in Elasticsearch 8.11 + // The semantic text field was added to the knowledge base index in 8.17 + // Indices created in 8.10 do not support semantic text field and need to be reindexed + // Failing: See https://github.com/elastic/kibana/issues/233043 + describe('Knowledge base: when upgrading from 8.10 to 8.18', function () { + // Intentionally skipped in all serverless environnments (local and MKI) + // because the migration scenario being tested is not relevant to MKI and Serverless. + this.tags(['skipServerless', 'skipCloud']); + + before(async () => { + // in a real environment we will use the ELSER inference endpoint (`.elser-2-elasticsearch`) which is pre-installed + // For testing purposes we will use the tiny ELSER model + + log.info('Setting up tiny ELSER model and inference endpoint'); + await setupTinyElserModelAndInferenceEndpoint(getService); + }); + + after(async () => { + log.info('Restoring index assets'); + await restoreIndexAssets(getService); + + log.info('Tearing down tiny ELSER model and inference endpoint'); + await teardownTinyElserModelAndInferenceEndpoint(getService); + }); + + describe('before running migrations', () => { + before(async () => { + log.info('Delete index assets'); + await deleteIndexAssets(getService); + + log.info('Restoring snapshot'); + await restoreKbSnapshot({ + log, + es, + snapshotFolderName: 'snapshot_kb_8.10', + snapshotName: 'my_snapshot', + }); + + log.info('Creating index assets'); + await createOrUpdateIndexAssets(observabilityAIAssistantAPIClient); + }); + + it('has an index created version earlier than 8.11', async () => { + await retry.try(async () => { + const indexCreatedVersion = await getKbIndexCreatedVersion(es); + expect(semver.lt(indexCreatedVersion, '8.11.0')).to.be(true); + }); + }); + + it('cannot add new entries to KB until reindex has completed', async () => { + const res1 = await createKnowledgeBaseEntry(); + + expect(res1.status).to.be(503); + expect((res1.body as unknown as Error).message).to.eql( + 'The index ".kibana-observability-ai-assistant-kb" does not support semantic text and must be reindexed. This re-index operation has been scheduled and will be started automatically. Please try again later.' + ); + + // wait for reindex to have updated the index + await retry.try(async () => { + const indexCreatedVersion = await getKbIndexCreatedVersion(es); + expect(semver.gte(indexCreatedVersion, '8.18.0')).to.be(true); + }); + + const res2 = await createKnowledgeBaseEntry(); + expect(res2.status).to.be(200); + }); + }); + + describe('after running migrations', () => { + beforeEach(async () => { + await deleteIndexAssets(getService); + await restoreKbSnapshot({ + log, + es, + snapshotFolderName: 'snapshot_kb_8.10', + snapshotName: 'my_snapshot', + }); + await createOrUpdateIndexAssets(observabilityAIAssistantAPIClient); + await runStartupMigrations(observabilityAIAssistantAPIClient); + }); + + it('has an index created version later than 8.18', async () => { + await retry.try(async () => { + const indexCreatedVersion = await getKbIndexCreatedVersion(es); + expect(semver.gt(indexCreatedVersion, '8.18.0')).to.be(true); + }); + }); + + it('can add new entries', async () => { + const { status } = await createKnowledgeBaseEntry(); + expect(status).to.be(200); + }); + + it('has default ELSER inference endpoint', async () => { + await retry.try(async () => { + const { body } = await observabilityAIAssistantAPIClient.editor({ + endpoint: 'GET /internal/observability_ai_assistant/kb/status', + }); + + expect(body.endpoint?.inference_id).to.eql(TINY_ELSER_INFERENCE_ID); + expect(body.endpoint?.service_settings.model_id).to.eql(TINY_ELSER_MODEL_ID); + }); + }); + + it('have a deployed model', async () => { + await retry.try(async () => { + const { body } = await observabilityAIAssistantAPIClient.editor({ + endpoint: 'GET /internal/observability_ai_assistant/kb/status', + }); + + expect(body.inferenceModelState === InferenceModelState.READY).to.be(true); + }); + }); + }); + + function createKnowledgeBaseEntry() { + const knowledgeBaseEntry = { + id: 'my-doc-id-1', + title: 'My title', + text: 'My content', + }; + + return observabilityAIAssistantAPIClient.editor({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', + params: { body: knowledgeBaseEntry }, + }); + } + }); +}