From 283d66370bf0bc90c38ac7369fe69a6e6910f5ea Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 17 Oct 2023 09:23:19 -0700 Subject: [PATCH 01/19] Switch heading types on setup page Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 56 +++++++++++-------- .../components/setup_integration.tsx | 12 ++-- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 65d9b65be..2aa23faf3 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -60,13 +60,15 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiSpacer euiSpacer--l" /> - -

+
- Integration Details -

- +

+ Integration Details +

+ +
- -

+
- Integration Connection -

- +

+ Integration Connection +

+
+
- -

+
- Integration Details -

- +

+ Integration Details +

+
+
- -

+
- Integration Connection -

- +

+ Integration Connection +

+
+
Set Up Integration - -

Integration Details

- + +

Integration Details

+ - -

Integration Connection

- + +

Integration Connection

+ Date: Mon, 23 Oct 2023 09:46:09 -0700 Subject: [PATCH 02/19] Add alb and nginx create table queries Signed-off-by: Simeon Widdis --- .../aws_elb/assets/create_table-1.0.0.sql | 40 +++++++++++++++++++ .../repository/aws_elb/aws_elb-1.0.0.json | 9 ++++- .../nginx/assets/create_table-1.0.0.sql | 17 ++++++++ .../repository/nginx/nginx-1.0.0.json | 9 ++++- 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql new file mode 100644 index 000000000..2ba99a955 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql @@ -0,0 +1,40 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + type string, + time string, + elb string, + client_ip string, + client_port int, + target_ip string, + target_port int, + request_processing_time double, + target_processing_time double, + response_processing_time double, + elb_status_code int, + target_status_code string, + received_bytes bigint, + sent_bytes bigint, + request_verb string, + request_url string, + request_proto string, + user_agent string, + ssl_cipher string, + ssl_protocol string, + target_group_arn string, + trace_id string, + domain_name string, + chosen_cert_arn string, + matched_rule_priority string, + request_creation_time string, + actions_executed string, + redirect_url string, + lambda_error_reason string, + target_port_list string, + target_status_code_list string, + classification string, + classification_reason string +) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' +WITH + SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' + ) LOCATION '{s3_bucket_location}'; \ No newline at end of file diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 3a8a58d29..2c68cb7e1 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -50,7 +50,14 @@ "savedObjects": { "name": "aws_elb", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "create_table", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json" diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql new file mode 100644 index 000000000..25a5ff1e1 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql @@ -0,0 +1,17 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( + remote_addr string, + remote_host string, + remote_user string, + time_local string, + request_method string, + request_path string, + status_code int, + body_bytes_sent int, + http_referer string, + http_user_agent string, +) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' +WITH + SERDEPROPERTIES ( + 'serialization.format' = '1', + 'input.regex' = '^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)")' + ) LOCATION '{s3_bucket_location}'; diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index daca06aa8..a52b2ce32 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -38,7 +38,14 @@ "savedObjects": { "name": "nginx", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "create_table", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json" From 725a4b4db0104a3200670fa5e096dd21920b45a4 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 12:04:20 -0700 Subject: [PATCH 03/19] Switch from toast to callout for set up failures Signed-off-by: Simeon Widdis --- .../components/create_integration_helpers.ts | 13 +++--- .../components/setup_integration.tsx | 44 ++++++++++++++++--- .../repository/fs_data_adaptor.ts | 15 +++++-- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 7e465c26b..1bebd558f 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -313,16 +313,13 @@ export async function addIntegrationRequest( .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { body: JSON.stringify({ name, dataSource }), }) - .then((_res) => { + .then((res) => { setToast(`${name} integration successfully added!`, 'success'); - window.location.hash = `#/installed/${_res.data?.id}`; + window.location.hash = `#/installed/${res.data?.id}`; return true; }) - .catch((_err) => { - setToast( - 'Failed to load integration. Check Added Integrations table for more details', - 'danger' - ); + .catch((err) => { + setToast('Failed to load integration', 'danger', err.message); return false; }); if (!addSample || !response) { @@ -333,7 +330,7 @@ export async function addIntegrationRequest( .then((res) => res.data) .catch((err) => { console.error(err); - setToast('The sample data could not be retrieved', 'danger'); + setToast('Failed to load integration', 'danger', 'The sample data could not be retrieved.'); return { sampleData: [] }; }); const requestBody = diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 11d41be35..0f38ac662 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -6,6 +6,7 @@ import { EuiBottomBar, EuiButton, + EuiCallOut, EuiComboBox, EuiFieldText, EuiFlexGroup, @@ -25,9 +26,9 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useState, useEffect } from 'react'; +import { Color } from 'common/constants/integrations'; import { coreRefs } from '../../../framework/core_refs'; import { IntegrationTemplate, addIntegrationRequest } from './create_integration_helpers'; -import { useToast } from '../../../../public/components/common/toast'; import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { DATACONNECTIONS_BASE } from '../../../../common/constants/shared'; @@ -38,10 +39,13 @@ export interface IntegrationSetupInputs { connectionLocation: string; } +type SetupCallout = { show: true; title: string; color?: Color; text?: string } | { show: false }; + interface IntegrationConfigProps { config: IntegrationSetupInputs; updateConfig: (updates: Partial) => void; integration: IntegrationTemplate; + setupCallout: SetupCallout; } // TODO support localization @@ -156,7 +160,10 @@ const runQuery = async ( trackProgress(3); return { ok: true, value: poll }; } else if (poll.status === 'FAILURE') { - return { ok: false, error: new Error('FAILURE status', { cause: poll }) }; + return { + ok: false, + error: new Error(poll.error ?? 'No error information provided', { cause: poll }), + }; } await sleep(3000); } @@ -170,6 +177,7 @@ export function SetupIntegrationForm({ config, updateConfig, integration, + setupCallout, }: IntegrationConfigProps) { const connectionType = INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType)!; @@ -194,6 +202,12 @@ export function SetupIntegrationForm({

Set Up Integration

+ {setupCallout.show ? ( + +

{setupCallout.text}

+ + ) : null} +

Integration Details

 @@ -262,6 +276,7 @@ export function SetupBottomBar({ setLoading, loadingProgress, setProgress, + setSetupCallout, }: { config: IntegrationSetupInputs; integration: IntegrationTemplate; @@ -269,8 +284,16 @@ export function SetupBottomBar({ setLoading: (loading: boolean) => void; loadingProgress: number; setProgress: (updater: number | ((progress: number) => number)) => void; + setSetupCallout: (setupCallout: SetupCallout) => void; }) { - const { setToast } = useToast(); + // Drop-in replacement for setToast + const setCalloutLikeToast = (title: string, color?: Color, text?: string) => + setSetupCallout({ + show: true, + title, + color, + text, + }); return ( @@ -305,7 +328,7 @@ export function SetupBottomBar({ integration.name, config.displayName, integration, - setToast, + setCalloutLikeToast, config.displayName, config.connectionDataSource ); @@ -331,9 +354,12 @@ export function SetupBottomBar({ setProgress(currProgress + step) ); if (!result.ok) { - console.error('Query failed', result.error); setLoading(false); - setToast('Something went wrong.', 'danger'); + setCalloutLikeToast( + 'Failed to add integration', + 'danger', + result.error.message + ); return; } } @@ -344,7 +370,7 @@ export function SetupBottomBar({ integration.name, config.displayName, integration, - setToast, + setCalloutLikeToast, config.displayName, config.connectionDataSource ); @@ -402,6 +428,8 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { assets: {}, } as IntegrationTemplate); + const [setupCallout, setSetupCallout] = useState({ show: false } as SetupCallout); + const [showLoading, setShowLoading] = useState(false); const [loadingProgress, setLoadingProgress] = useState(0); @@ -430,6 +458,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { config={integConfig} updateConfig={updateConfig} integration={template} + setupCallout={setupCallout} /> )} @@ -441,6 +470,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { setLoading={setShowLoading} loadingProgress={loadingProgress} setProgress={setLoadingProgress} + setSetupCallout={setSetupCallout} /> diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts index df1c6781a..b6915d5ee 100644 --- a/server/adaptors/integrations/repository/fs_data_adaptor.ts +++ b/server/adaptors/integrations/repository/fs_data_adaptor.ts @@ -48,6 +48,15 @@ function tryParseNDJson(content: string): object[] | null { } } +// Check if a location is a directory without an exception if location not found +const safeIsDirectory = async (maybeDirectory: string): Promise => { + try { + return (await fs.lstat(maybeDirectory)).isDirectory(); + } catch (_err: unknown) { + return false; + } +}; + /** * A CatalogDataAdaptor that reads from the local filesystem. * Used to read Integration information when the user uploads their own catalog. @@ -133,15 +142,13 @@ export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { } async getDirectoryType(dirname?: string): Promise<'integration' | 'repository' | 'unknown'> { - const isDir = (await fs.lstat(path.join(this.directory, dirname ?? '.'))).isDirectory(); + const isDir = await safeIsDirectory(path.join(this.directory, dirname ?? '.')); if (!isDir) { return 'unknown'; } // Sloppily just check for one mandatory integration directory to distinguish. // Improve if we need to distinguish a repository with an integration named "schemas". - const hasSchemas = ( - await fs.lstat(path.join(this.directory, dirname ?? '.', 'schemas')) - ).isDirectory(); + const hasSchemas = await safeIsDirectory(path.join(this.directory, dirname ?? '.', 'schemas')); return hasSchemas ? 'integration' : 'repository'; } From 75d45103c7f6c9b8d9c6a3d4616aadc873b9ba85 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 13:25:01 -0700 Subject: [PATCH 04/19] Fix label selection for truncated labels Signed-off-by: Simeon Widdis --- .../available_integration_overview_page.tsx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/public/components/integrations/components/available_integration_overview_page.tsx b/public/components/integrations/components/available_integration_overview_page.tsx index 360bf0db4..cd9a8300b 100644 --- a/public/components/integrations/components/available_integration_overview_page.tsx +++ b/public/components/integrations/components/available_integration_overview_page.tsx @@ -32,9 +32,9 @@ export interface AvailableIntegrationType { version?: string | undefined; displayName?: string; integrationType: string; - statics: any; - components: any[]; - displayAssets: any[]; + statics: unknown; + components: Array<{ name: string }>; + displayAssets: unknown[]; } export interface AvailableIntegrationsTableProps { @@ -183,7 +183,11 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver ? AvailableIntegrationsCardView({ data: { hits: data.hits.filter((hit) => - helper.every((compon) => hit.components.map((x) => x.name).includes(compon)) + helper.every((compon) => + hit.components + .map((x) => x.name.split('_').findLast(() => true)) + .includes(compon) + ) ), }, isCardView, @@ -197,7 +201,11 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver loading: false, data: { hits: data.hits.filter((hit) => - helper.every((compon) => hit.components.map((x) => x.name).includes(compon)) + helper.every((compon) => + hit.components + .map((x) => x.name.split('_').findLast(() => true)) + .includes(compon) + ) ), }, isCardView, From f89bc708dbf9c954eda7f1a167ec7b8b04051274 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 13:46:25 -0700 Subject: [PATCH 05/19] Fix button color Signed-off-by: Simeon Widdis --- .../integrations/components/setup_integration.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 0f38ac662..bb2956db7 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -6,6 +6,7 @@ import { EuiBottomBar, EuiButton, + EuiButtonEmpty, EuiCallOut, EuiComboBox, EuiFieldText, @@ -299,8 +300,8 @@ export function SetupBottomBar({ - { // TODO evil hack because props aren't set up @@ -311,7 +312,7 @@ export function SetupBottomBar({ }} > Discard - + Date: Mon, 23 Oct 2023 13:51:57 -0700 Subject: [PATCH 06/19] Fix tests Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 122 +++++++++--------- .../__tests__/setup_integration.test.tsx | 1 + 2 files changed, 65 insertions(+), 58 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 2aa23faf3..e3615782a 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -42,6 +42,11 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = "type": "", } } + setupCallout={ + Object { + "show": false, + } + } updateConfig={[Function]} > @@ -60,6 +65,11 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiSpacer euiSpacer--l" /> + +
+
@@ -707,11 +718,11 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = class="euiFlexItem euiFlexItem--flexGrowZero" > - - + + + +
@@ -1045,6 +1046,11 @@ exports[`Integration Setup Page Renders the form as expected 1`] = ` className="euiSpacer euiSpacer--l" /> + +
+
{ config={TEST_INTEGRATION_SETUP_INPUTS} updateConfig={() => {}} integration={TEST_INTEGRATION_CONFIG} + setupCallout={{ show: false }} /> ); From 3c30aba8caf404c32e9cb901e8390303e34087f6 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 14:11:58 -0700 Subject: [PATCH 07/19] Remove loading progress bar Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 2 - .../components/setup_integration.tsx | 45 +++++-------------- 2 files changed, 10 insertions(+), 37 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index e3615782a..900d6c694 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -692,9 +692,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = } } loading={false} - loadingProgress={0} setLoading={[Function]} - setProgress={[Function]} setSetupCallout={[Function]} > diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index bb2956db7..6cf68116c 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -9,12 +9,14 @@ import { EuiButtonEmpty, EuiCallOut, EuiComboBox, + EuiEmptyPrompt, EuiFieldText, EuiFlexGroup, EuiFlexItem, EuiForm, EuiFormRow, EuiLoadingDashboards, + EuiLoadingLogo, EuiModal, EuiPage, EuiPageBody, @@ -275,16 +277,12 @@ export function SetupBottomBar({ integration, loading, setLoading, - loadingProgress, - setProgress, setSetupCallout, }: { config: IntegrationSetupInputs; integration: IntegrationTemplate; loading: boolean; setLoading: (loading: boolean) => void; - loadingProgress: number; - setProgress: (updater: number | ((progress: number) => number)) => void; setSetupCallout: (setupCallout: SetupCallout) => void; }) { // Drop-in replacement for setToast @@ -333,14 +331,12 @@ export function SetupBottomBar({ config.displayName, config.connectionDataSource ); - setProgress((progress) => progress + 1); } else if (config.connectionType === 's3') { const http = coreRefs.http!; const assets = await http.get( `${INTEGRATIONS_BASE}/repository/${integration.name}/assets` ); - setProgress((progress) => progress + 1); // Queries must exist because we disable s3 if they're not present for (const query of assets.data.queries!) { @@ -350,10 +346,7 @@ export function SetupBottomBar({ ); queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation); queryStr = queryStr.replaceAll('{object_name}', integration.name); - const currProgress = loadingProgress; // Need a frozen copy for getting accurate query steps - const result = await runQuery(queryStr, (step) => - setProgress(currProgress + step) - ); + const result = await runQuery(queryStr, (_) => {}); if (!result.ok) { setLoading(false); setCalloutLikeToast( @@ -375,7 +368,6 @@ export function SetupBottomBar({ config.displayName, config.connectionDataSource ); - setProgress((progress) => progress + 1); } else { console.error('Invalid data source type'); } @@ -390,27 +382,14 @@ export function SetupBottomBar({ ); } -export function LoadingPage({ value, max }: { value: number; max: number }) { +export function LoadingPage() { return ( <> - - - - - - - -

Adding Integration

- - - - - This may take a few minutes. The integration and assets are being added. - - - - - + } + title={

Setting Up the Integration

} + body={

This can take several minutes.

} + /> ); } @@ -432,7 +411,6 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { const [setupCallout, setSetupCallout] = useState({ show: false } as SetupCallout); const [showLoading, setShowLoading] = useState(false); - const [loadingProgress, setLoadingProgress] = useState(0); useEffect(() => { const getTemplate = async () => { @@ -445,7 +423,6 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { const updateConfig = (updates: Partial) => setConfig(Object.assign({}, integConfig, updates)); - const maxProgress = 2 + 3 * (template.assets?.queries?.length ?? 0); return ( @@ -453,7 +430,7 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { {showLoading ? ( - + ) : ( From 78fa19c15e26f389aa231b06d7b696616ed0d0c1 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 14:35:42 -0700 Subject: [PATCH 08/19] Remove unused imports Signed-off-by: Simeon Widdis --- .../components/integrations/components/setup_integration.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 6cf68116c..831aa0ad9 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -15,14 +15,11 @@ import { EuiFlexItem, EuiForm, EuiFormRow, - EuiLoadingDashboards, EuiLoadingLogo, - EuiModal, EuiPage, EuiPageBody, EuiPageContent, EuiPageContentBody, - EuiProgress, EuiSelect, EuiSpacer, EuiText, From 9f13cf08fef377778b008e6f5232b67f6492cfa0 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 23 Oct 2023 15:05:47 -0700 Subject: [PATCH 09/19] Refactor labels to make distinctions more semantically useful Signed-off-by: Simeon Widdis --- .../available_integration_overview_page.tsx | 20 +++++-------------- .../repository/apache/apache-1.0.0.json | 2 +- .../aws_cloudfront/aws_cloudfront-1.0.0.json | 2 +- .../aws_cloudtrail/aws_cloudtrail-1.0.0.json | 2 +- .../repository/aws_elb/aws_elb-1.0.0.json | 2 +- .../repository/aws_rds/aws_rds-1.0.0.json | 2 +- .../repository/aws_s3/aws_s3-1.0.0.json | 2 +- .../aws_vpc_flow/aws_vpc_flow-1.0.0.json | 2 +- .../repository/aws_waf/aws_waf-1.0.0.json | 2 +- .../__data__/repository/k8s/k8s-1.0.0.json | 2 +- .../repository/nginx/nginx-1.0.0.json | 2 +- server/adaptors/integrations/validators.ts | 1 + 12 files changed, 16 insertions(+), 25 deletions(-) diff --git a/public/components/integrations/components/available_integration_overview_page.tsx b/public/components/integrations/components/available_integration_overview_page.tsx index cd9a8300b..9fc1ec9ea 100644 --- a/public/components/integrations/components/available_integration_overview_page.tsx +++ b/public/components/integrations/components/available_integration_overview_page.tsx @@ -117,7 +117,9 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver http.get(`${INTEGRATIONS_BASE}/repository`).then((exists) => { setData(exists.data); - let newItems = exists.data.hits.flatMap((hit: { labels?: string[] }) => hit.labels ?? []); + let newItems = exists.data.hits.flatMap( + (hit: { labels?: string[] }) => hit.labels?.sort() ?? [] + ); newItems = [...new Set(newItems)].sort().map((newItem) => { return { name: newItem, @@ -182,13 +184,7 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver {isCardView ? AvailableIntegrationsCardView({ data: { - hits: data.hits.filter((hit) => - helper.every((compon) => - hit.components - .map((x) => x.name.split('_').findLast(() => true)) - .includes(compon) - ) - ), + hits: data.hits.filter((hit) => helper.every((tag) => hit.labels?.includes(tag))), }, isCardView, setCardView, @@ -200,13 +196,7 @@ export function AvailableIntegrationOverviewPage(props: AvailableIntegrationOver : AvailableIntegrationsTable({ loading: false, data: { - hits: data.hits.filter((hit) => - helper.every((compon) => - hit.components - .map((x) => x.name.split('_').findLast(() => true)) - .includes(compon) - ) - ), + hits: data.hits.filter((hit) => helper.every((tag) => hit.labels?.includes(tag))), }, isCardView, setCardView, diff --git a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json index 80cbfc906..99fb24a16 100644 --- a/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/apache/apache-1.0.0.json @@ -5,7 +5,7 @@ "description": "Apache web logs collector", "license": "Apache-2.0", "type": "logs_apache", - "labels": ["log", "communication", "http"], + "labels": ["Observability", "Logs"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/apache/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json index 5a1c526e5..2f4d806b2 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudfront/aws_cloudfront-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS cloudfront Object Store", "license": "Apache-2.0", "type": "logs-aws_cloudfront", - "labels": ["log", "aws", "s3", "cloud", "cloudfront"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudfront/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json index 3964ae11c..818b6e8a9 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS CloudTrail log collector", "license": "Apache-2.0", "type": "logs-aws_cloudtrail", - "labels": ["log", "aws", "s3", "cloud", "cloudtrail"], + "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 3a8a58d29..a91ee5bc9 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS Elastic Load Balancer collector", "license": "Apache-2.0", "type": "logs_elb", - "labels": ["log", "aws", "communication", "http", "cloud", "elb", "url"], + "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_elb/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json index c34f2697b..ca46d616c 100644 --- a/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_rds/aws_rds-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS RDS", "license": "Apache-2.0", "type": "logs_rds", - "labels": ["log", "aws", "s3", "cloud", "rds"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_rds/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json index 60a1b011d..c519e57b7 100644 --- a/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_s3/aws_s3-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS S3 Object Store", "license": "Apache-2.0", "type": "logs_s3", - "labels": ["log", "aws", "s3", "cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_s3/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index deee7d362..24110600a 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS VPC Flow log collector", "license": "Apache-2.0", "type": "logs_vpc", - "labels": ["log", "aws", "s3", "cloud", "communication", "vpc"], + "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json index 1b1171eb9..2075aecf6 100644 --- a/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_waf/aws_waf-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS waf log collector", "license": "Apache-2.0", "type": "logs_waf", - "labels": ["log", "aws", "s3", "cloud", "waf"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_waf/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json b/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json index 7454b25a9..d359ffe8e 100644 --- a/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/k8s/k8s-1.0.0.json @@ -5,7 +5,7 @@ "description": "Kubernetes web logs collector", "license": "Apache-2.0", "type": "logs-k8s", - "labels": ["log", "k8s", "cloud", "container"], + "labels": ["Observability", "Logs", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/k8s/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index daca06aa8..1b87c497a 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -5,7 +5,7 @@ "description": "Nginx HTTP server collector", "license": "Apache-2.0", "type": "logs", - "labels": ["log", "http", "communication"], + "labels": ["Observability", "Logs", "Flint S3"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", "statics": { diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 2a4a717a9..5aa65cf81 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -26,6 +26,7 @@ const templateSchema: JSONSchemaType = { license: { type: 'string' }, type: { type: 'string' }, labels: { type: 'array', items: { type: 'string' }, nullable: true }, + tags: { type: 'array', items: { type: 'string' }, nullable: true }, author: { type: 'string', nullable: true }, description: { type: 'string', nullable: true }, sourceUrl: { type: 'string', nullable: true }, From 1a0161799db47158d163a8a6bf55a59960885f29 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 11:36:06 -0700 Subject: [PATCH 10/19] Add running queries for ELB integration Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 62 ++++++++++++------- .../aws_cloudtrail/aws_cloudtrail-1.0.0.json | 2 +- .../aws_elb/assets/create_mv-1.0.0.sql | 38 ++++++++++++ .../aws_elb/assets/create_table-1.0.0.sql | 17 +++-- .../aws_elb/assets/refresh_mv-1.0.0.sql | 1 + .../repository/aws_elb/aws_elb-1.0.0.json | 10 +++ .../aws_vpc_flow/aws_vpc_flow-1.0.0.json | 2 +- .../repository/nginx/nginx-1.0.0.json | 2 +- 8 files changed, 97 insertions(+), 37 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index d9f89ae8f..299976557 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -60,9 +60,9 @@ const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< [ 's3', { - title: 'Table', - lower: 'table', - help: 'Select a table to pull the data from.', + title: 'Catalog', + lower: 'catalog', + help: 'Select a catalog to pull the data from.', }, ], [ @@ -126,8 +126,9 @@ const suggestDataSources = async (type: string): Promise void -): Promise> => { + datasource: string, + sessionId: string | null +): Promise> => { // Used for polling const sleep = (ms: number) => { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -135,15 +136,14 @@ const runQuery = async ( try { const http = coreRefs.http!; - const queryId = ( - await http.post(CONSOLE_PROXY, { - body: JSON.stringify({ query, lang: 'sql' }), - query: { - path: '_plugins/_async_query', - method: 'POST', - }, - }) - ).queryId; + const queryResponse: { queryId: string; sessionId: string } = await http.post(CONSOLE_PROXY, { + body: JSON.stringify({ query, datasource, lang: 'sql', sessionId }), + query: { + path: '_plugins/_async_query', + method: 'POST', + }, + }); + const [queryId, newSessionId] = [queryResponse.queryId, queryResponse.sessionId]; while (true) { const poll = await http.post(CONSOLE_PROXY, { body: '{}', @@ -152,13 +152,14 @@ const runQuery = async ( method: 'GET', }, }); - if (poll.status === 'PENDING') { - trackProgress(1); - } else if (poll.status === 'RUNNING') { - trackProgress(2); - } else if (poll.status === 'SUCCESS') { - trackProgress(3); - return { ok: true, value: poll }; + if (poll.status === 'SUCCESS') { + return { + ok: true, + value: { + poll, + sessionId: newSessionId, + }, + }; } else if (poll.status === 'FAILURE') { return { ok: false, @@ -254,6 +255,16 @@ export function SetupIntegrationForm({ }} selectedOptions={[{ label: config.connectionDataSource }]} singleSelection={{ asPlainText: true }} + onCreateOption={(searchValue) => { + const normalizedSearchValue = searchValue.trim(); + if (!normalizedSearchValue) { + return; + } + const newOption = { label: normalizedSearchValue }; + setDataSourceSuggestions((ds) => ds.concat([newOption])); + updateConfig({ connectionDataSource: newOption.label }); + }} + customOptionText={`Select {searchValue} as your ${connectionType.lower}`} /> {config.connectionType === 's3' ? ( @@ -305,6 +316,7 @@ export function SetupBottomBar({ hash = hash.substring(0, hash.lastIndexOf('/setup')); window.location.hash = hash; }} + disabled={loading} > Discard @@ -318,6 +330,7 @@ export function SetupBottomBar({ disabled={config.displayName.length < 1 || config.connectionDataSource.length < 1} onClick={async () => { setLoading(true); + let sessionId: string | null = null; if (config.connectionType === 'index') { await addIntegrationRequest( @@ -344,7 +357,8 @@ export function SetupBottomBar({ ); queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation); queryStr = queryStr.replaceAll('{object_name}', integration.name); - const result = await runQuery(queryStr, (_) => {}); + queryStr = queryStr.replaceAll(/\s+/g, ' '); + const result = await runQuery(queryStr, config.connectionDataSource, sessionId); if (!result.ok) { setLoading(false); setCalloutLikeToast( @@ -354,6 +368,7 @@ export function SetupBottomBar({ ); return; } + sessionId = result.value.sessionId ?? sessionId; } // Once everything is ready, add the integration to the new datasource as usual // TODO determine actual values here after more about queries is known @@ -364,7 +379,7 @@ export function SetupBottomBar({ integration, setCalloutLikeToast, config.displayName, - config.connectionDataSource + `flint_${config.connectionDataSource}_default_${integration.name}_mview` ); } else { console.error('Invalid data source type'); @@ -407,7 +422,6 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { } as IntegrationTemplate); const [setupCallout, setSetupCallout] = useState({ show: false } as SetupCallout); - const [showLoading, setShowLoading] = useState(false); useEffect(() => { diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json index 818b6e8a9..41dbc35e8 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS CloudTrail log collector", "license": "Apache-2.0", "type": "logs-aws_cloudtrail", - "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql new file mode 100644 index 000000000..87e89bb3e --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql @@ -0,0 +1,38 @@ +CREATE MATERIALIZED VIEW + {table_name}_mview AS + SELECT + type as `aws.elb.elb_type`, + time as `@timestamp`, + elb as `aws.elb.elb_name`, + client_ip as `aws.elb.client.ip`, + client_port as `aws.elb.client.port`, + target_ip as `aws.elb.target_ip`, + target_port as `aws.elb.target_port`, + request_processing_time as `aws.elb.request_processing_time`, + target_processing_time as `aws.elb.target_processing_time`, + response_processing_time as `aws.elb.response_processing_time`, + elb_status_code as `aws.elb.elb_status_code`, + target_status_code as `aws.elb.target_status_code`, + received_bytes as `aws.elb.received_bytes`, + sent_bytes as `aws.elb.sent_bytes`, + request_verb as `http.request.method`, + request_url as `url.full`, + request_proto as `url.schema`, + user_agent as `http.user_agent.name`, + ssl_cipher as `aws.elb.ssl_cipher`, + ssl_protocol as `aws.elb.ssl_protocol`, + target_group_arn as `aws.elb.target_group_arn`, + trace_id as `traceId`, + domain_name as `url.domain`, + chosen_cert_arn as `aws.elb.chosen_cert_arn`, + matched_rule_priority as `aws.elb.matched_rule_priority`, + request_creation_time as `aws.elb.request_creation_time`, + actions_executed as `aws.elb.actions_executed`, + redirect_url as `aws.elb.redirect_url`, + lambda_error_reason as `aws.elb.lambda_error_reason`, + target_port_list as `aws.elb.target_port_list`, + target_status_code_list as `aws.elb.target_status_code_list`, + classification as `aws.elb.classification`, + classification_reason as `aws.elb.classification_reason` + FROM + {table_name}; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql index 2ba99a955..5ab0861ce 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql @@ -1,15 +1,15 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( type string, - time string, + time timestamp, elb string, client_ip string, - client_port int, + client_port bigint, target_ip string, - target_port int, + target_port bigint, request_processing_time double, target_processing_time double, response_processing_time double, - elb_status_code int, + elb_status_code bigint, target_status_code string, received_bytes bigint, sent_bytes bigint, @@ -32,9 +32,6 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( target_status_code_list string, classification string, classification_reason string -) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' -WITH - SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = '([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"' - ) LOCATION '{s3_bucket_location}'; \ No newline at end of file +) +USING parquet +LOCATION '{s3_bucket_location}'; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql new file mode 100644 index 000000000..0f6ba9896 --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql @@ -0,0 +1 @@ +REFRESH MATERIALIZED VIEW {table_name}_mview; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 32be8b28b..27480a5d0 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -56,6 +56,16 @@ "name": "create_table", "version": "1.0.0", "language": "sql" + }, + { + "name": "create_mv", + "version": "1.0.0", + "language": "sql" + }, + { + "name": "refresh_mv", + "version": "1.0.0", + "language": "sql" } ] }, diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index 24110600a..1d92e2665 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -5,7 +5,7 @@ "description": "AWS VPC Flow log collector", "license": "Apache-2.0", "type": "logs_vpc", - "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], + "labels": ["Observability", "Logs", "AWS", "Cloud"], "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", "statics": { diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 784b5d641..7ab1cf437 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -5,7 +5,7 @@ "description": "Nginx HTTP server collector", "license": "Apache-2.0", "type": "logs", - "labels": ["Observability", "Logs", "Flint S3"], + "labels": ["Observability", "Logs"], "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info", "statics": { From 47cc88b23a0a106229c1ed080f5480e7cb508f8f Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 11:39:41 -0700 Subject: [PATCH 11/19] Update tests Signed-off-by: Simeon Widdis --- .../__tests__/__snapshots__/setup_integration.test.tsx.snap | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index e8f7bee82..65371e88b 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -354,12 +354,14 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = aria-describedby="random_html_id-help-0" async={false} compressed={false} + customOptionText="Select {searchValue} as your index" fullWidth={false} id="random_html_id" isClearable={true} isLoading={true} onBlur={[Function]} onChange={[Function]} + onCreateOption={[Function]} onFocus={[Function]} options={Array []} selectedOptions={ @@ -817,6 +819,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = > @@ -1354,12 +1357,14 @@ exports[`Integration Setup Page Renders the form as expected 1`] = ` aria-describedby="random_html_id-help-0" async={false} compressed={false} + customOptionText="Select {searchValue} as your index" fullWidth={false} id="random_html_id" isClearable={true} isLoading={true} onBlur={[Function]} onChange={[Function]} + onCreateOption={[Function]} onFocus={[Function]} options={Array []} selectedOptions={ From e7ffde6ec06ab9043fea5c52d526ebc3be4280f0 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 13:09:36 -0700 Subject: [PATCH 12/19] Improve error handling for integration creation Signed-off-by: Simeon Widdis --- .../components/create_integration_helpers.ts | 13 +++++++++---- .../components/setup_integration.tsx | 18 ++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 1bebd558f..543f68828 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -296,7 +296,7 @@ export async function addIntegrationRequest( setToast: (title: string, color?: Color, text?: string | undefined) => void, name?: string, dataSource?: string -) { +): Promise { const http = coreRefs.http!; if (addSample) { createDataSourceMappings( @@ -309,7 +309,7 @@ export async function addIntegrationRequest( dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; } - const response: boolean = await http + let response: boolean = await http .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { body: JSON.stringify({ name, dataSource }), }) @@ -323,7 +323,7 @@ export async function addIntegrationRequest( return false; }); if (!addSample || !response) { - return; + return response; } const data: { sampleData: unknown[] } = await http .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) @@ -337,7 +337,7 @@ export async function addIntegrationRequest( data.sampleData .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) .join('\n') + '\n'; - http + response = await http .post(CONSOLE_PROXY, { body: requestBody, query: { @@ -345,8 +345,13 @@ export async function addIntegrationRequest( method: 'POST', }, }) + .then((_) => { + return true; + }) .catch((err) => { console.error(err); setToast('Failed to load sample data', 'danger'); + return false; }); + return response; } diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 299976557..7ee94f336 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -145,14 +145,14 @@ const runQuery = async ( }); const [queryId, newSessionId] = [queryResponse.queryId, queryResponse.sessionId]; while (true) { - const poll = await http.post(CONSOLE_PROXY, { + const poll: { status: string; error?: string } = await http.post(CONSOLE_PROXY, { body: '{}', query: { path: '_plugins/_async_query/' + queryId, method: 'GET', }, }); - if (poll.status === 'SUCCESS') { + if (poll.status.toLowerCase() === 'success') { return { ok: true, value: { @@ -160,7 +160,8 @@ const runQuery = async ( sessionId: newSessionId, }, }; - } else if (poll.status === 'FAILURE') { + // Fail status can inconsistently be "failed" or "failure" + } else if (poll.status.toLowerCase().startsWith('fail')) { return { ok: false, error: new Error(poll.error ?? 'No error information provided', { cause: poll }), @@ -333,7 +334,7 @@ export function SetupBottomBar({ let sessionId: string | null = null; if (config.connectionType === 'index') { - await addIntegrationRequest( + const res = await addIntegrationRequest( false, integration.name, config.displayName, @@ -342,6 +343,9 @@ export function SetupBottomBar({ config.displayName, config.connectionDataSource ); + if (!res) { + setLoading(false); + } } else if (config.connectionType === 's3') { const http = coreRefs.http!; @@ -372,7 +376,7 @@ export function SetupBottomBar({ } // Once everything is ready, add the integration to the new datasource as usual // TODO determine actual values here after more about queries is known - await addIntegrationRequest( + const res = await addIntegrationRequest( false, integration.name, config.displayName, @@ -381,10 +385,12 @@ export function SetupBottomBar({ config.displayName, `flint_${config.connectionDataSource}_default_${integration.name}_mview` ); + if (!res) { + setLoading(false); + } } else { console.error('Invalid data source type'); } - setLoading(false); }} > Add Integration From 23cd257b1fead7d8062e0613d5eb347027170e64 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 13:22:51 -0700 Subject: [PATCH 13/19] Resolve missed merge Signed-off-by: Simeon Widdis --- .../repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json index ff80cd204..41dbc35e8 100644 --- a/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json @@ -5,11 +5,7 @@ "description": "AWS CloudTrail log collector", "license": "Apache-2.0", "type": "logs-aws_cloudtrail", -<<<<<<< HEAD "labels": ["Observability", "Logs", "AWS", "Cloud"], -======= - "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], ->>>>>>> upstream/main "author": "OpenSearch", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info", "statics": { From dba790ba1a9e83682c7f2a121a45471e2e21613c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 13:24:12 -0700 Subject: [PATCH 14/19] Fix another missed merge marker Signed-off-by: Simeon Widdis --- .../__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json index d301e3c16..1d92e2665 100644 --- a/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json @@ -5,11 +5,7 @@ "description": "AWS VPC Flow log collector", "license": "Apache-2.0", "type": "logs_vpc", -<<<<<<< HEAD "labels": ["Observability", "Logs", "AWS", "Cloud"], -======= - "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"], ->>>>>>> upstream/main "author": "Haidong Wang", "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info", "statics": { From 744e7b377cd0c9e4c3c91daa43d9d07af4dc3b94 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 13:25:03 -0700 Subject: [PATCH 15/19] Remove buggy table from nginx Signed-off-by: Simeon Widdis --- .../nginx/assets/create_table-1.0.0.sql | 17 ----------------- .../__data__/repository/nginx/nginx-1.0.0.json | 9 +-------- 2 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql deleted file mode 100644 index 25a5ff1e1..000000000 --- a/server/adaptors/integrations/__data__/repository/nginx/assets/create_table-1.0.0.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( - remote_addr string, - remote_host string, - remote_user string, - time_local string, - request_method string, - request_path string, - status_code int, - body_bytes_sent int, - http_referer string, - http_user_agent string, -) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.RegexSerDe' -WITH - SERDEPROPERTIES ( - 'serialization.format' = '1', - 'input.regex' = '^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(?[^\]]*)\] "(?\S+)(?: +(?[^\"]*?)(?: +\S*)?)?" (?[^ ]*) (?[^ ]*)(?: "(?[^\"]*)" "(?[^\"]*)")' - ) LOCATION '{s3_bucket_location}'; diff --git a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json index 7ab1cf437..975430d0d 100644 --- a/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json @@ -38,14 +38,7 @@ "savedObjects": { "name": "nginx", "version": "1.0.0" - }, - "queries": [ - { - "name": "create_table", - "version": "1.0.0", - "language": "sql" - } - ] + } }, "sampleData": { "path": "sample.json" From 08867b56547bf9787c874c37f721274814582f02 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 26 Oct 2023 15:04:42 -0700 Subject: [PATCH 16/19] Improve validation for integration setup Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 30 ++++++++++- .../components/setup_integration.tsx | 53 +++++++++++++++---- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 65371e88b..3688bcb34 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -31,6 +31,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = Object { "connectionDataSource": "", "connectionLocation": "", + "connectionTableName": "sample", "connectionType": "index", "displayName": "sample Integration", } @@ -87,9 +88,15 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = @@ -101,12 +108,15 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiFormRow__labelWrapper" > updateConfig({ connectionLocation: event.target.value })} placeholder="s3://" - isInvalid={!config.connectionLocation.startsWith('s3://')} + isInvalid={isBlurred && !config.connectionLocation.startsWith('s3://')} + onBlur={() => { + setIsBlurred(true); + }} /> From 0c266a65ed8beeae5dae251f761cd492faa9c3b5 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 27 Oct 2023 11:40:28 -0700 Subject: [PATCH 19/19] Update queries for parsing raw logs Signed-off-by: Simeon Widdis --- .../aws_elb/assets/create_mv-1.0.0.sql | 68 +++++++++---------- .../aws_elb/assets/create_table-1.0.0.sql | 17 +++-- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql index 87e89bb3e..8520ae8b7 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql @@ -1,38 +1,38 @@ CREATE MATERIALIZED VIEW {table_name}_mview AS SELECT - type as `aws.elb.elb_type`, - time as `@timestamp`, - elb as `aws.elb.elb_name`, - client_ip as `aws.elb.client.ip`, - client_port as `aws.elb.client.port`, - target_ip as `aws.elb.target_ip`, - target_port as `aws.elb.target_port`, - request_processing_time as `aws.elb.request_processing_time`, - target_processing_time as `aws.elb.target_processing_time`, - response_processing_time as `aws.elb.response_processing_time`, - elb_status_code as `aws.elb.elb_status_code`, - target_status_code as `aws.elb.target_status_code`, - received_bytes as `aws.elb.received_bytes`, - sent_bytes as `aws.elb.sent_bytes`, - request_verb as `http.request.method`, - request_url as `url.full`, - request_proto as `url.schema`, - user_agent as `http.user_agent.name`, - ssl_cipher as `aws.elb.ssl_cipher`, - ssl_protocol as `aws.elb.ssl_protocol`, - target_group_arn as `aws.elb.target_group_arn`, - trace_id as `traceId`, - domain_name as `url.domain`, - chosen_cert_arn as `aws.elb.chosen_cert_arn`, - matched_rule_priority as `aws.elb.matched_rule_priority`, - request_creation_time as `aws.elb.request_creation_time`, - actions_executed as `aws.elb.actions_executed`, - redirect_url as `aws.elb.redirect_url`, - lambda_error_reason as `aws.elb.lambda_error_reason`, - target_port_list as `aws.elb.target_port_list`, - target_status_code_list as `aws.elb.target_status_code_list`, - classification as `aws.elb.classification`, - classification_reason as `aws.elb.classification_reason` + type as `aws.elb.elb_type`, + time as `@timestamp`, + elb as `aws.elb.elb_name`, + split_part(client_ip, ':', 1) as `aws.elb.client.ip`, + split_part(client_ip, ':', 2) as `aws.elb.client.port`, + split_part(target_ip, ':', 1) as `aws.elb.target.ip`, + split_part(target_ip, ':', 2) as `aws.elb.target.port`, + request_processing_time as `aws.elb.request_processing_time`, + target_processing_time as `aws.elb.target_processing_time`, + response_processing_time as `aws.elb.response_processing_time`, + elb_status_code as `aws.elb.elb_status_code`, + target_status_code as `aws.elb.target_status_code`, + received_bytes as `aws.elb.received_bytes`, + sent_bytes as `aws.elb.sent_bytes`, + split_part(request, ' ', 1) as `http.request.method`, + split_part(request, ' ', 2) as `url.full`, + split_part(request, ' ', 3) as `url.schema`, + user_agent as `http.user_agent.name`, + ssl_cipher as `aws.elb.ssl_cipher`, + ssl_protocol as `aws.elb.ssl_protocol`, + target_group_arn as `aws.elb.target_group_arn`, + trace_id as `traceId`, + domain_name as `url.domain`, + chosen_cert_arn as `aws.elb.chosen_cert_arn`, + matched_rule_priority as `aws.elb.matched_rule_priority`, + request_creation_time as `aws.elb.request_creation_time`, + actions_executed as `aws.elb.actions_executed`, + redirect_url as `aws.elb.redirect_url`, + lambda_error_reason as `aws.elb.lambda_error_reason`, + target_port_list as `aws.elb.target_port_list`, + target_status_code_list as `aws.elb.target_status_code_list`, + classification as `aws.elb.classification`, + classification_reason as `aws.elb.classification_reason` FROM - {table_name}; + {table_name}; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql index 5ab0861ce..3a5ed53f2 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql @@ -3,19 +3,15 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( time timestamp, elb string, client_ip string, - client_port bigint, target_ip string, - target_port bigint, request_processing_time double, target_processing_time double, response_processing_time double, - elb_status_code bigint, + elb_status_code int, target_status_code string, received_bytes bigint, sent_bytes bigint, - request_verb string, - request_url string, - request_proto string, + request string, user_agent string, ssl_cipher string, ssl_protocol string, @@ -24,7 +20,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( domain_name string, chosen_cert_arn string, matched_rule_priority string, - request_creation_time string, + request_creation_time timestamp, actions_executed string, redirect_url string, lambda_error_reason string, @@ -33,5 +29,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} ( classification string, classification_reason string ) -USING parquet -LOCATION '{s3_bucket_location}'; +USING csv +LOCATION '{s3_bucket_location}' +OPTIONS ( + sep=' ' +);