Add integrations queries for Flint

public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap

@@ -354,12 +354,14 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
  aria-describedby="random_html_id-help-0"
  async={false}
  compressed={false}
+ customOptionText="Select {searchValue} as your index"
  fullWidth={false}
  id="random_html_id"
  isClearable={true}
  isLoading={true}
  onBlur={[Function]}
  onChange={[Function]}
+ onCreateOption={[Function]}
  onFocus={[Function]}
  options={Array []}
  selectedOptions={
@@ -817,6 +819,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
  >
  <EuiButtonEmpty
  color="text"
+ disabled={false}
  iconType="cross"
  onClick={[Function]}
  >
@@ -1354,12 +1357,14 @@ exports[`Integration Setup Page Renders the form as expected 1`] = `
  aria-describedby="random_html_id-help-0"
  async={false}
  compressed={false}
+ customOptionText="Select {searchValue} as your index"
  fullWidth={false}
  id="random_html_id"
  isClearable={true}
  isLoading={true}
  onBlur={[Function]}
  onChange={[Function]}
+ onCreateOption={[Function]}
  onFocus={[Function]}
  options={Array []}
  selectedOptions={

public/components/integrations/components/create_integration_helpers.ts

@@ -296,7 +296,7 @@ export async function addIntegrationRequest(
  setToast: (title: string, color?: Color, text?: string | undefined) => void,
  name?: string,
  dataSource?: string
-) {
+): Promise<boolean> {
  const http = coreRefs.http!;
  if (addSample) {
  createDataSourceMappings(
@@ -309,7 +309,7 @@ export async function addIntegrationRequest(
  dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`;
  }
 
- const response: boolean = await http
+ let response: boolean = await http
  .post(`${INTEGRATIONS_BASE}/store/${templateName}`, {
  body: JSON.stringify({ name, dataSource }),
  })
@@ -323,7 +323,7 @@ export async function addIntegrationRequest(
  return false;
  });
  if (!addSample || !response) {
- return;
+ return response;
  }
  const data: { sampleData: unknown[] } = await http
  .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`)
@@ -337,16 +337,21 @@ export async function addIntegrationRequest(
  data.sampleData
  .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`)
  .join('\n') + '\n';
- http
+ response = await http
  .post(CONSOLE_PROXY, {
  body: requestBody,
  query: {
  path: `${dataSource}/_bulk?refresh=wait_for`,
  method: 'POST',
  },
  })
+ .then((_) => {
+ return true;
+ })
  .catch((err) => {
  console.error(err);
  setToast('Failed to load sample data', 'danger');
+ return false;
  });
+ return response;
 }

public/components/integrations/components/setup_integration.tsx

@@ -60,9 +60,9 @@ const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map<
  [
  's3',
  {
- title: 'Table',
- lower: 'table',
- help: 'Select a table to pull the data from.',
+ title: 'Catalog',
+ lower: 'catalog',
+ help: 'Select a catalog to pull the data from.',
  },
  ],
  [
@@ -126,40 +126,42 @@ const suggestDataSources = async (type: string): Promise<Array<{ label: string }
 
 const runQuery = async (
  query: string,
- trackProgress: (step: number) => void
-): Promise<Result<object>> => {
+ datasource: string,
+ sessionId: string | null
+): Promise<Result<{ poll: object; sessionId: string }>> => {
  // Used for polling
  const sleep = (ms: number) => {
  return new Promise((resolve) => setTimeout(resolve, ms));
  };
 
  try {
  const http = coreRefs.http!;
- const queryId = (
- await http.post(CONSOLE_PROXY, {
- body: JSON.stringify({ query, lang: 'sql' }),
- query: {
- path: '_plugins/_async_query',
- method: 'POST',
- },
- })
- ).queryId;
+ const queryResponse: { queryId: string; sessionId: string } = await http.post(CONSOLE_PROXY, {
+ body: JSON.stringify({ query, datasource, lang: 'sql', sessionId }),
+ query: {
+ path: '_plugins/_async_query',
+ method: 'POST',
+ },
+ });
+ const [queryId, newSessionId] = [queryResponse.queryId, queryResponse.sessionId];
  while (true) {
- const poll = await http.post(CONSOLE_PROXY, {
+ const poll: { status: string; error?: string } = await http.post(CONSOLE_PROXY, {
  body: '{}',
  query: {
  path: '_plugins/_async_query/' + queryId,
  method: 'GET',
  },
  });
- if (poll.status === 'PENDING') {
- trackProgress(1);
- } else if (poll.status === 'RUNNING') {
- trackProgress(2);
- } else if (poll.status === 'SUCCESS') {
- trackProgress(3);
- return { ok: true, value: poll };
- } else if (poll.status === 'FAILURE') {
+ if (poll.status.toLowerCase() === 'success') {
+ return {
+ ok: true,
+ value: {
+ poll,
+ sessionId: newSessionId,
+ },
+ };
+ // Fail status can inconsistently be "failed" or "failure"
+ } else if (poll.status.toLowerCase().startsWith('fail')) {
  return {
  ok: false,
  error: new Error(poll.error ?? 'No error information provided', { cause: poll }),
@@ -254,6 +256,16 @@ export function SetupIntegrationForm({
  }}
  selectedOptions={[{ label: config.connectionDataSource }]}
  singleSelection={{ asPlainText: true }}
+ onCreateOption={(searchValue) => {
+ const normalizedSearchValue = searchValue.trim();
+ if (!normalizedSearchValue) {
+ return;
+ }
+ const newOption = { label: normalizedSearchValue };
+ setDataSourceSuggestions((ds) => ds.concat([newOption]));
+ updateConfig({ connectionDataSource: newOption.label });
+ }}
+ customOptionText={`Select {searchValue} as your ${connectionType.lower}`}
  />
  </EuiFormRow>
  {config.connectionType === 's3' ? (
@@ -305,6 +317,7 @@ export function SetupBottomBar({
  hash = hash.substring(0, hash.lastIndexOf('/setup'));
  window.location.hash = hash;
  }}
+ disabled={loading}
  >
  Discard
  </EuiButtonEmpty>
@@ -318,9 +331,10 @@ export function SetupBottomBar({
  disabled={config.displayName.length < 1 || config.connectionDataSource.length < 1}
  onClick={async () => {
  setLoading(true);
+ let sessionId: string | null = null;
 
  if (config.connectionType === 'index') {
- await addIntegrationRequest(
+ const res = await addIntegrationRequest(
  false,
  integration.name,
  config.displayName,
@@ -329,6 +343,9 @@ export function SetupBottomBar({
  config.displayName,
  config.connectionDataSource
  );
+ if (!res) {
+ setLoading(false);
+ }
  } else if (config.connectionType === 's3') {
  const http = coreRefs.http!;
 
@@ -344,7 +361,8 @@ export function SetupBottomBar({
  );
  queryStr = queryStr.replaceAll('{s3_bucket_location}', config.connectionLocation);
  queryStr = queryStr.replaceAll('{object_name}', integration.name);
- const result = await runQuery(queryStr, (_) => {});
+ queryStr = queryStr.replaceAll(/\s+/g, ' ');
+ const result = await runQuery(queryStr, config.connectionDataSource, sessionId);
  if (!result.ok) {
  setLoading(false);
  setCalloutLikeToast(
@@ -354,22 +372,25 @@ export function SetupBottomBar({
  );
  return;
  }
+ sessionId = result.value.sessionId ?? sessionId;
  }
  // Once everything is ready, add the integration to the new datasource as usual
  // TODO determine actual values here after more about queries is known
- await addIntegrationRequest(
+ const res = await addIntegrationRequest(
  false,
  integration.name,
  config.displayName,
  integration,
  setCalloutLikeToast,
  config.displayName,
- config.connectionDataSource
+ `flint_${config.connectionDataSource}_default_${integration.name}_mview`
  );
+ if (!res) {
+ setLoading(false);
+ }
  } else {
  console.error('Invalid data source type');
  }
- setLoading(false);
  }}
  >
  Add Integration
@@ -407,7 +428,6 @@ export function SetupIntegrationPage({ integration }: { integration: string }) {
  } as IntegrationTemplate);
 
  const [setupCallout, setSetupCallout] = useState({ show: false } as SetupCallout);
-
  const [showLoading, setShowLoading] = useState(false);
 
  useEffect(() => {

server/adaptors/integrations/__data__/repository/aws_cloudtrail/aws_cloudtrail-1.0.0.json

@@ -5,7 +5,7 @@
  "description": "AWS CloudTrail log collector",
  "license": "Apache-2.0",
  "type": "logs-aws_cloudtrail",
- "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
+ "labels": ["Observability", "Logs", "AWS", "Cloud"],
  "author": "OpenSearch",
  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_cloudtrail/info",
  "statics": {

server/adaptors/integrations/__data__/repository/aws_elb/assets/create_mv-1.0.0.sql

@@ -0,0 +1,38 @@
+CREATE MATERIALIZED VIEW
+ {table_name}_mview AS
+ SELECT
+ type as `aws.elb.elb_type`,
+ time as `@timestamp`,
+ elb as `aws.elb.elb_name`,
+ client_ip as `aws.elb.client.ip`,
+ client_port as `aws.elb.client.port`,
+ target_ip as `aws.elb.target_ip`,
+ target_port as `aws.elb.target_port`,
+ request_processing_time as `aws.elb.request_processing_time`,
+ target_processing_time as `aws.elb.target_processing_time`,
+ response_processing_time as `aws.elb.response_processing_time`,
+ elb_status_code as `aws.elb.elb_status_code`,
+ target_status_code as `aws.elb.target_status_code`,
+ received_bytes as `aws.elb.received_bytes`,
+ sent_bytes as `aws.elb.sent_bytes`,
+ request_verb as `http.request.method`,
+ request_url as `url.full`,
+ request_proto as `url.schema`,
+ user_agent as `http.user_agent.name`,
+ ssl_cipher as `aws.elb.ssl_cipher`,
+ ssl_protocol as `aws.elb.ssl_protocol`,
+ target_group_arn as `aws.elb.target_group_arn`,
+ trace_id as `traceId`,
+ domain_name as `url.domain`,
+ chosen_cert_arn as `aws.elb.chosen_cert_arn`,
+ matched_rule_priority as `aws.elb.matched_rule_priority`,
+ request_creation_time as `aws.elb.request_creation_time`,
+ actions_executed as `aws.elb.actions_executed`,
+ redirect_url as `aws.elb.redirect_url`,
+ lambda_error_reason as `aws.elb.lambda_error_reason`,
+ target_port_list as `aws.elb.target_port_list`,
+ target_status_code_list as `aws.elb.target_status_code_list`,
+ classification as `aws.elb.classification`,
+ classification_reason as `aws.elb.classification_reason`
+ FROM
+ {table_name};

server/adaptors/integrations/__data__/repository/aws_elb/assets/create_table-1.0.0.sql

@@ -0,0 +1,37 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
+ type string,
+ time timestamp,
+ elb string,
+ client_ip string,
+ client_port bigint,
+ target_ip string,
+ target_port bigint,
+ request_processing_time double,
+ target_processing_time double,
+ response_processing_time double,
+ elb_status_code bigint,
+ target_status_code string,
+ received_bytes bigint,
+ sent_bytes bigint,
+ request_verb string,
+ request_url string,
+ request_proto string,
+ user_agent string,
+ ssl_cipher string,
+ ssl_protocol string,
+ target_group_arn string,
+ trace_id string,
+ domain_name string,
+ chosen_cert_arn string,
+ matched_rule_priority string,
+ request_creation_time string,
+ actions_executed string,
+ redirect_url string,
+ lambda_error_reason string,
+ target_port_list string,
+ target_status_code_list string,
+ classification string,
+ classification_reason string
+)
+USING parquet
+LOCATION '{s3_bucket_location}';

server/adaptors/integrations/__data__/repository/aws_elb/assets/refresh_mv-1.0.0.sql

@@ -0,0 +1 @@
+REFRESH MATERIALIZED VIEW {table_name}_mview;

server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json

@@ -50,7 +50,24 @@
  "savedObjects": {
  "name": "aws_elb",
  "version": "1.0.0"
- }
+ },
+ "queries": [
+ {
+ "name": "create_table",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "create_mv",
+ "version": "1.0.0",
+ "language": "sql"
+ },
+ {
+ "name": "refresh_mv",
+ "version": "1.0.0",
+ "language": "sql"
+ }
+ ]
  },
  "sampleData": {
  "path": "sample.json"

server/adaptors/integrations/__data__/repository/aws_vpc_flow/aws_vpc_flow-1.0.0.json

@@ -5,7 +5,7 @@
  "description": "AWS VPC Flow log collector",
  "license": "Apache-2.0",
  "type": "logs_vpc",
- "labels": ["Observability", "Logs", "AWS", "Flint S3", "Cloud"],
+ "labels": ["Observability", "Logs", "AWS", "Cloud"],
  "author": "Haidong Wang",
  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/aws_vpc_flow/info",
  "statics": {

server/adaptors/integrations/__data__/repository/nginx/nginx-1.0.0.json

@@ -5,7 +5,7 @@
  "description": "Nginx HTTP server collector",
  "license": "Apache-2.0",
  "type": "logs",
- "labels": ["Observability", "Logs", "Flint S3"],
+ "labels": ["Observability", "Logs"],
  "author": "OpenSearch",
  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/nginx/info",
  "statics": {