From dee41313c737a0259ba13494d476d9baec52bdfe Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Mon, 17 Oct 2022 14:34:42 +0500 Subject: [PATCH 1/6] Fix performance issue with autocomplete suggestions --- .../lib/autocomplete_entities/mapping.ts | 16 +---- .../register_get_route.ts | 70 ++++++++++++++++--- 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/src/plugins/console/public/lib/autocomplete_entities/mapping.ts b/src/plugins/console/public/lib/autocomplete_entities/mapping.ts index ddb6905fa6e53..71e72dac0a280 100644 --- a/src/plugins/console/public/lib/autocomplete_entities/mapping.ts +++ b/src/plugins/console/public/lib/autocomplete_entities/mapping.ts @@ -121,23 +121,9 @@ export class Mapping implements BaseMapping { }; loadMappings = (mappings: IndicesGetMappingResponse) => { - const maxMappingSize = Object.keys(mappings).length > 10 * 1024 * 1024; - let mappingsResponse; - if (maxMappingSize) { - // eslint-disable-next-line no-console - console.warn( - `Mapping size is larger than 10MB (${ - Object.keys(mappings).length / 1024 / 1024 - } MB). Ignoring...` - ); - mappingsResponse = {}; - } else { - mappingsResponse = mappings; - } - this.perIndexTypes = {}; - Object.entries(mappingsResponse).forEach(([index, indexMapping]) => { + Object.entries(mappings).forEach(([index, indexMapping]) => { const normalizedIndexMappings: Record = {}; let transformedMapping: Record = indexMapping; diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts index 9d5778f0a9b0f..9f43fe7bc30aa 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts @@ -7,6 +7,7 @@ */ import type { IScopedClusterClient } from '@kbn/core/server'; import { parse } from 'query-string'; +import type { IncomingMessage } from 'http'; import type { RouteDependencies } from '../../..'; import { API_BASE_PATH } from '../../../../../common/constants'; @@ -17,9 +18,41 @@ interface Settings { dataStreams: boolean; } +const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; + +function streamToString(stream: IncomingMessage, limit: number) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => { + chunks.push(chunk); + if (Buffer.byteLength(Buffer.concat(chunks)) > limit) { + stream.destroy(); + reject(new Error('Response size limit exceeded')); + } + }); + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + stream.on('error', reject); + }); +} + +async function limitEntityResponseSize(esClient: IScopedClusterClient, entity: string) { + const stream = await esClient.asInternalUser.transport.request( + { + method: 'GET', + path: `/${entity}`, + }, + { asStream: true } + ); + + // Limit the response size to 10MB, because the response can be very large and sending it to the client + // can cause the browser to hang. + const body = await streamToString(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); + return JSON.parse(body); +} + async function getMappings(esClient: IScopedClusterClient, settings: Settings) { if (settings.fields) { - return esClient.asInternalUser.indices.getMapping(); + return limitEntityResponseSize(esClient, '_mapping'); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return Promise.resolve({}); @@ -27,7 +60,7 @@ async function getMappings(esClient: IScopedClusterClient, settings: Settings) { async function getAliases(esClient: IScopedClusterClient, settings: Settings) { if (settings.indices) { - return esClient.asInternalUser.indices.getAlias(); + return limitEntityResponseSize(esClient, '_alias'); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return Promise.resolve({}); @@ -35,7 +68,7 @@ async function getAliases(esClient: IScopedClusterClient, settings: Settings) { async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) { if (settings.dataStreams) { - return esClient.asInternalUser.indices.getDataStream(); + return limitEntityResponseSize(esClient, '_data_stream'); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return Promise.resolve({}); @@ -44,9 +77,9 @@ async function getDataStreams(esClient: IScopedClusterClient, settings: Settings async function getTemplates(esClient: IScopedClusterClient, settings: Settings) { if (settings.templates) { return Promise.all([ - esClient.asInternalUser.indices.getTemplate(), - esClient.asInternalUser.indices.getIndexTemplate(), - esClient.asInternalUser.cluster.getComponentTemplate(), + limitEntityResponseSize(esClient, '_template'), + limitEntityResponseSize(esClient, '_component_template'), + limitEntityResponseSize(esClient, '_index_template'), ]); } // If the user doesn't want autocomplete suggestions, then clear any that exist. @@ -71,11 +104,26 @@ export function registerGetRoute({ router, lib: { handleEsError } }: RouteDepend } const esClient = (await ctx.core).elasticsearch.client; - const mappings = await getMappings(esClient, settings); - const aliases = await getAliases(esClient, settings); - const dataStreams = await getDataStreams(esClient, settings); - const [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}] = - await getTemplates(esClient, settings); + + // Wait for all requests to complete, in case one of them fails return the successfull ones + const results = await Promise.allSettled([ + getMappings(esClient, settings), + getAliases(esClient, settings), + getDataStreams(esClient, settings), + getTemplates(esClient, settings), + ]); + + const [ + mappings, + aliases, + dataStreams, + [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}], + ] = results.map((result) => { + if (result.status === 'fulfilled') { + return result.value; + } + return {}; + }); return response.ok({ body: { From 24999177e9c89f04dda34de1094486f5ed25f7b1 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Mon, 17 Oct 2022 15:48:13 +0500 Subject: [PATCH 2/6] Add unit tests for streamToString function --- src/plugins/console/server/lib/utils/index.ts | 1 + .../server/lib/utils/streamToString.test.ts | 35 +++++++++++++++++++ .../server/lib/utils/streamToString.ts | 24 +++++++++++++ .../register_get_route.ts | 16 +-------- 4 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 src/plugins/console/server/lib/utils/streamToString.test.ts create mode 100644 src/plugins/console/server/lib/utils/streamToString.ts diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index 0da3e36b575e3..0651735c9b4e4 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -8,3 +8,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; +export { streamToString } from './streamToString'; diff --git a/src/plugins/console/server/lib/utils/streamToString.test.ts b/src/plugins/console/server/lib/utils/streamToString.test.ts new file mode 100644 index 0000000000000..ea2d3aa6cbeaa --- /dev/null +++ b/src/plugins/console/server/lib/utils/streamToString.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Readable } from 'stream'; +import { streamToString } from './streamToString'; +import type { IncomingMessage } from 'http'; + +describe('streamToString', () => { + it('should limit the response size', async () => { + const stream = new Readable({ + read() { + this.push('a'.repeat(1000)); + }, + }); + await expect( + streamToString(stream as IncomingMessage, 500) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`); + }); + + it('should resolve with string', async () => { + const stream = new Readable({ + read() { + this.push('{"test": "test"}'); + this.push(null); + }, + }); + const result = await streamToString(stream as IncomingMessage, 5000); + expect(result).toEqual('{"test": "test"}'); + }); +}); diff --git a/src/plugins/console/server/lib/utils/streamToString.ts b/src/plugins/console/server/lib/utils/streamToString.ts new file mode 100644 index 0000000000000..200709d3962ac --- /dev/null +++ b/src/plugins/console/server/lib/utils/streamToString.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IncomingMessage } from 'http'; + +export function streamToString(stream: IncomingMessage, limit: number) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => { + chunks.push(chunk); + if (Buffer.byteLength(Buffer.concat(chunks)) > limit) { + stream.destroy(); + reject(new Error('Response size limit exceeded')); + } + }); + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + stream.on('error', reject); + }); +} diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts index 9f43fe7bc30aa..ce5920499ab35 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts @@ -10,6 +10,7 @@ import { parse } from 'query-string'; import type { IncomingMessage } from 'http'; import type { RouteDependencies } from '../../..'; import { API_BASE_PATH } from '../../../../../common/constants'; +import { streamToString } from '../../../../lib/utils'; interface Settings { indices: boolean; @@ -20,21 +21,6 @@ interface Settings { const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; -function streamToString(stream: IncomingMessage, limit: number) { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on('data', (chunk) => { - chunks.push(chunk); - if (Buffer.byteLength(Buffer.concat(chunks)) > limit) { - stream.destroy(); - reject(new Error('Response size limit exceeded')); - } - }); - stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); - stream.on('error', reject); - }); -} - async function limitEntityResponseSize(esClient: IScopedClusterClient, entity: string) { const stream = await esClient.asInternalUser.transport.request( { From 9d070ccd9253aae39f044799f01fe714825c7ae4 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Mon, 17 Oct 2022 15:59:15 +0500 Subject: [PATCH 3/6] Fix checks --- src/plugins/console/server/lib/utils/index.ts | 2 +- .../utils/{streamToString.test.ts => stream_to_string.test.ts} | 2 +- .../server/lib/utils/{streamToString.ts => stream_to_string.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/plugins/console/server/lib/utils/{streamToString.test.ts => stream_to_string.test.ts} (95%) rename src/plugins/console/server/lib/utils/{streamToString.ts => stream_to_string.ts} (100%) diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index 0651735c9b4e4..b601d6844b239 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -8,4 +8,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; -export { streamToString } from './streamToString'; +export { streamToString } from './stream_to_string'; diff --git a/src/plugins/console/server/lib/utils/streamToString.test.ts b/src/plugins/console/server/lib/utils/stream_to_string.test.ts similarity index 95% rename from src/plugins/console/server/lib/utils/streamToString.test.ts rename to src/plugins/console/server/lib/utils/stream_to_string.test.ts index ea2d3aa6cbeaa..c031e77da2c25 100644 --- a/src/plugins/console/server/lib/utils/streamToString.test.ts +++ b/src/plugins/console/server/lib/utils/stream_to_string.test.ts @@ -7,7 +7,7 @@ */ import { Readable } from 'stream'; -import { streamToString } from './streamToString'; +import { streamToString } from './stream_to_string'; import type { IncomingMessage } from 'http'; describe('streamToString', () => { diff --git a/src/plugins/console/server/lib/utils/streamToString.ts b/src/plugins/console/server/lib/utils/stream_to_string.ts similarity index 100% rename from src/plugins/console/server/lib/utils/streamToString.ts rename to src/plugins/console/server/lib/utils/stream_to_string.ts From b3bd51bd09f2b662e208d0588799c421308c23c0 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Mon, 17 Oct 2022 16:42:15 +0500 Subject: [PATCH 4/6] Fix an edge case --- .../register_get_route.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts index ce5920499ab35..3cea12c0020a0 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts @@ -99,18 +99,28 @@ export function registerGetRoute({ router, lib: { handleEsError } }: RouteDepend getTemplates(esClient, settings), ]); - const [ - mappings, - aliases, - dataStreams, - [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}], - ] = results.map((result) => { + const map = results.map((result, idx) => { + // If the request was successful, return the result if (result.status === 'fulfilled') { return result.value; } + + // getTemplates returns an array of results, so we need to handle it in case of failure + if (idx === results.length - 1) { + return []; + } + + // If the request failed, return an empty object return {}; }); + const [ + mappings, + aliases, + dataStreams, + [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}], + ] = map; + return response.ok({ body: { mappings, From 0f7f5d8a3986ab32af225c2bf47cb03b6326b646 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Tue, 18 Oct 2022 11:30:12 +0500 Subject: [PATCH 5/6] Refactor the changes to be more readable --- .../register_get_route.ts | 99 +++++++++++-------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts index 3cea12c0020a0..6784e5eab48d7 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts @@ -20,56 +20,77 @@ interface Settings { } const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; +// Limit the response size to 10MB, because the response can be very large and sending it to the client +// can cause the browser to hang. -async function limitEntityResponseSize(esClient: IScopedClusterClient, entity: string) { - const stream = await esClient.asInternalUser.transport.request( - { - method: 'GET', - path: `/${entity}`, - }, - { asStream: true } - ); - - // Limit the response size to 10MB, because the response can be very large and sending it to the client - // can cause the browser to hang. - const body = await streamToString(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); - return JSON.parse(body); +function streamToJSON(stream: IncomingMessage) { + return streamToString(stream, RESPONSE_SIZE_LIMIT).then((response) => JSON.parse(response)); } async function getMappings(esClient: IScopedClusterClient, settings: Settings) { if (settings.fields) { - return limitEntityResponseSize(esClient, '_mapping'); + const stream = await esClient.asInternalUser.indices.getMapping(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); } // If the user doesn't want autocomplete suggestions, then clear any that exist. - return Promise.resolve({}); + return {}; } async function getAliases(esClient: IScopedClusterClient, settings: Settings) { if (settings.indices) { - return limitEntityResponseSize(esClient, '_alias'); + const stream = await esClient.asInternalUser.indices.getAlias(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); } // If the user doesn't want autocomplete suggestions, then clear any that exist. - return Promise.resolve({}); + return {}; } async function getDataStreams(esClient: IScopedClusterClient, settings: Settings) { if (settings.dataStreams) { - return limitEntityResponseSize(esClient, '_data_stream'); + const stream = await esClient.asInternalUser.indices.getDataStream(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Settings) { + if (settings.templates) { + const stream = await esClient.asInternalUser.indices.getTemplate(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); + } + // If the user doesn't want autocomplete suggestions, then clear any that exist. + return {}; +} + +async function getComponentTemplates(esClient: IScopedClusterClient, settings: Settings) { + if (settings.templates) { + const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); } // If the user doesn't want autocomplete suggestions, then clear any that exist. - return Promise.resolve({}); + return {}; } -async function getTemplates(esClient: IScopedClusterClient, settings: Settings) { +async function getIndexTemplates(esClient: IScopedClusterClient, settings: Settings) { if (settings.templates) { - return Promise.all([ - limitEntityResponseSize(esClient, '_template'), - limitEntityResponseSize(esClient, '_component_template'), - limitEntityResponseSize(esClient, '_index_template'), - ]); + const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, { + asStream: true, + }); + return streamToJSON(stream as unknown as IncomingMessage); } // If the user doesn't want autocomplete suggestions, then clear any that exist. - return Promise.resolve([]); + return {}; } export function registerGetRoute({ router, lib: { handleEsError } }: RouteDependencies) { @@ -96,31 +117,27 @@ export function registerGetRoute({ router, lib: { handleEsError } }: RouteDepend getMappings(esClient, settings), getAliases(esClient, settings), getDataStreams(esClient, settings), - getTemplates(esClient, settings), + getLegacyTemplates(esClient, settings), + getIndexTemplates(esClient, settings), + getComponentTemplates(esClient, settings), ]); - const map = results.map((result, idx) => { + const [ + mappings, + aliases, + dataStreams, + legacyTemplates, + indexTemplates, + componentTemplates, + ] = results.map((result) => { // If the request was successful, return the result if (result.status === 'fulfilled') { return result.value; } - - // getTemplates returns an array of results, so we need to handle it in case of failure - if (idx === results.length - 1) { - return []; - } - // If the request failed, return an empty object return {}; }); - const [ - mappings, - aliases, - dataStreams, - [legacyTemplates = {}, indexTemplates = {}, componentTemplates = {}], - ] = map; - return response.ok({ body: { mappings, From e9a538909318dd0aebe1ed87248d62c7cd366f5a Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov Date: Wed, 19 Oct 2022 11:21:33 +0500 Subject: [PATCH 6/6] Address CR change --- src/plugins/console/server/lib/utils/index.ts | 2 +- ...o_string.test.ts => stream_to_json.test.ts} | 10 +++++----- .../{stream_to_string.ts => stream_to_json.ts} | 7 +++++-- .../register_get_route.ts | 18 +++++++----------- 4 files changed, 18 insertions(+), 19 deletions(-) rename src/plugins/console/server/lib/utils/{stream_to_string.test.ts => stream_to_json.test.ts} (75%) rename src/plugins/console/server/lib/utils/{stream_to_string.ts => stream_to_json.ts} (78%) diff --git a/src/plugins/console/server/lib/utils/index.ts b/src/plugins/console/server/lib/utils/index.ts index b601d6844b239..2c595640eefbf 100644 --- a/src/plugins/console/server/lib/utils/index.ts +++ b/src/plugins/console/server/lib/utils/index.ts @@ -8,4 +8,4 @@ export { encodePath } from './encode_path'; export { toURL } from './to_url'; -export { streamToString } from './stream_to_string'; +export { streamToJSON } from './stream_to_json'; diff --git a/src/plugins/console/server/lib/utils/stream_to_string.test.ts b/src/plugins/console/server/lib/utils/stream_to_json.test.ts similarity index 75% rename from src/plugins/console/server/lib/utils/stream_to_string.test.ts rename to src/plugins/console/server/lib/utils/stream_to_json.test.ts index c031e77da2c25..780d3adb4a145 100644 --- a/src/plugins/console/server/lib/utils/stream_to_string.test.ts +++ b/src/plugins/console/server/lib/utils/stream_to_json.test.ts @@ -7,7 +7,7 @@ */ import { Readable } from 'stream'; -import { streamToString } from './stream_to_string'; +import { streamToJSON } from './stream_to_json'; import type { IncomingMessage } from 'http'; describe('streamToString', () => { @@ -18,18 +18,18 @@ describe('streamToString', () => { }, }); await expect( - streamToString(stream as IncomingMessage, 500) + streamToJSON(stream as IncomingMessage, 500) ).rejects.toThrowErrorMatchingInlineSnapshot(`"Response size limit exceeded"`); }); - it('should resolve with string', async () => { + it('should parse the response', async () => { const stream = new Readable({ read() { this.push('{"test": "test"}'); this.push(null); }, }); - const result = await streamToString(stream as IncomingMessage, 5000); - expect(result).toEqual('{"test": "test"}'); + const result = await streamToJSON(stream as IncomingMessage, 5000); + expect(result).toEqual({ test: 'test' }); }); }); diff --git a/src/plugins/console/server/lib/utils/stream_to_string.ts b/src/plugins/console/server/lib/utils/stream_to_json.ts similarity index 78% rename from src/plugins/console/server/lib/utils/stream_to_string.ts rename to src/plugins/console/server/lib/utils/stream_to_json.ts index 200709d3962ac..5ff1974e9fd47 100644 --- a/src/plugins/console/server/lib/utils/stream_to_string.ts +++ b/src/plugins/console/server/lib/utils/stream_to_json.ts @@ -8,7 +8,7 @@ import type { IncomingMessage } from 'http'; -export function streamToString(stream: IncomingMessage, limit: number) { +export function streamToJSON(stream: IncomingMessage, limit: number) { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; stream.on('data', (chunk) => { @@ -18,7 +18,10 @@ export function streamToString(stream: IncomingMessage, limit: number) { reject(new Error('Response size limit exceeded')); } }); - stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + stream.on('end', () => { + const response = Buffer.concat(chunks).toString('utf8'); + resolve(JSON.parse(response)); + }); stream.on('error', reject); }); } diff --git a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts index 6784e5eab48d7..98bd51e901b09 100644 --- a/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts +++ b/src/plugins/console/server/routes/api/console/autocomplete_entities/register_get_route.ts @@ -10,7 +10,7 @@ import { parse } from 'query-string'; import type { IncomingMessage } from 'http'; import type { RouteDependencies } from '../../..'; import { API_BASE_PATH } from '../../../../../common/constants'; -import { streamToString } from '../../../../lib/utils'; +import { streamToJSON } from '../../../../lib/utils'; interface Settings { indices: boolean; @@ -23,16 +23,12 @@ const RESPONSE_SIZE_LIMIT = 10 * 1024 * 1024; // Limit the response size to 10MB, because the response can be very large and sending it to the client // can cause the browser to hang. -function streamToJSON(stream: IncomingMessage) { - return streamToString(stream, RESPONSE_SIZE_LIMIT).then((response) => JSON.parse(response)); -} - async function getMappings(esClient: IScopedClusterClient, settings: Settings) { if (settings.fields) { const stream = await esClient.asInternalUser.indices.getMapping(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; @@ -43,7 +39,7 @@ async function getAliases(esClient: IScopedClusterClient, settings: Settings) { const stream = await esClient.asInternalUser.indices.getAlias(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; @@ -54,7 +50,7 @@ async function getDataStreams(esClient: IScopedClusterClient, settings: Settings const stream = await esClient.asInternalUser.indices.getDataStream(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; @@ -65,7 +61,7 @@ async function getLegacyTemplates(esClient: IScopedClusterClient, settings: Sett const stream = await esClient.asInternalUser.indices.getTemplate(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; @@ -76,7 +72,7 @@ async function getComponentTemplates(esClient: IScopedClusterClient, settings: S const stream = await esClient.asInternalUser.cluster.getComponentTemplate(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {}; @@ -87,7 +83,7 @@ async function getIndexTemplates(esClient: IScopedClusterClient, settings: Setti const stream = await esClient.asInternalUser.indices.getIndexTemplate(undefined, { asStream: true, }); - return streamToJSON(stream as unknown as IncomingMessage); + return streamToJSON(stream as unknown as IncomingMessage, RESPONSE_SIZE_LIMIT); } // If the user doesn't want autocomplete suggestions, then clear any that exist. return {};