[Feature] Datasource selector of multiple datasources (#5167)

* initial commit for datasource Signed-off-by: Eric <menwe@amazon.com> * initial commit for datasource service Signed-off-by: Eric <menwe@amazon.com> * [Data Sources] Move datasource codebase from datasource plugin to data plugin Signed-off-by: Eric <menwe@amazon.com> * [Data Sources] Add datasource factory Signed-off-by: Eric <menwe@amazon.com> * datasource service Signed-off-by: Eric <menwe@amazon.com> * datasource with factory Signed-off-by: Eric <menwe@amazon.com> * datasource selector Signed-off-by: Eric <menwe@amazon.com> * exposes datasources from data plugin Signed-off-by: Eric <menwe@amazon.com> * index pattern datasource registration Signed-off-by: Eric <menwe@amazon.com> * add datasource selector to discover Signed-off-by: Eric <menwe@amazon.com> * remove unused files Signed-off-by: Eric <menwe@amazon.com> * add/exposes types for datasources as a set of services Signed-off-by: Eric <menwe@amazon.com> * add datasource selector unit tests Signed-off-by: Eric <menwe@amazon.com> * add metadata to selectable Signed-off-by: Eric <menwe@amazon.com> * redirection to observability for non-index-pattern datasource Signed-off-by: Eric <menwe@amazon.com> * add datasource factory tests Signed-off-by: Eric <menwe@amazon.com> * add datasource service test Signed-off-by: Eric <menwe@amazon.com> * datasources related fixes Signed-off-by: Eric <menwe@amazon.com> * add tests for datasource selectable Signed-off-by: Eric <menwe@amazon.com> * added types/interfaces for sidebar selector, and add a couple of enhancements Signed-off-by: Eric <menwe@amazon.com> * remove pill effect Signed-off-by: Eric <menwe@amazon.com> * change type for name display in selector for index patterns Signed-off-by: Eric <menwe@amazon.com> * remove legacy index selector Signed-off-by: Eric <menwe@amazon.com> * rename datasource change handler Signed-off-by: Eric <menwe@amazon.com> * remove unused constants Signed-off-by: Eric <menwe@amazon.com> * fix a redirection bug Signed-off-by: Eric <menwe@amazon.com> * add to change log Signed-off-by: Eric <menwe@amazon.com> * fix bg color issue in source selector Signed-off-by: Eric <menwe@amazon.com> * address oui and missing guard Signed-off-by: Eric <menwe@amazon.com> * add test subj Signed-off-by: Eric <menwe@amazon.com> * i18 and datasource interface default return Signed-off-by: Eric <menwe@amazon.com> * add default datasource tests Signed-off-by: Eric <menwe@amazon.com> * fix typo Signed-off-by: Eric <menwe@amazon.com> * modify wording Signed-off-by: Eric <menwe@amazon.com> * add experimental annotation to datasource Signed-off-by: Eric <menwe@amazon.com> * add type datasource filtering to remove type error hint Signed-off-by: Eric <menwe@amazon.com> * remove unused type Signed-off-by: Eric <menwe@amazon.com> * add 'type' to option type Signed-off-by: Eric <menwe@amazon.com> * Type fixes Signed-off-by: Ashwin P Chandran <ashwinpc@amazon.com> * cherry-pick type changes from remote and add partial fixes Signed-off-by: Eric <menwe@amazon.com> * add one type exports Signed-off-by: Eric <menwe@amazon.com> * remaining type errors fixed Signed-off-by: Ashwin P Chandran <ashwinpc@amazon.com> * addressing comments Signed-off-by: Eric <menwe@amazon.com> * address dedup Signed-off-by: Eric <menwe@amazon.com> * refactor datasource_selectable to address comments Signed-off-by: Joshua Li <joshuali925@gmail.com> * remove unnecessary optional chaining Signed-off-by: Joshua Li <joshuali925@gmail.com> * refactor variable names Signed-off-by: Joshua Li <joshuali925@gmail.com> * move functnions out of selectable component Signed-off-by: Eric <menwe@amazon.com> * add comments for dedup/options list updates Signed-off-by: Eric <menwe@amazon.com> * add experimental annotation Signed-off-by: Eric <menwe@amazon.com> * callback and experimental annotation on types Signed-off-by: Eric <menwe@amazon.com> * handleSourceSelection callback Signed-off-by: Eric <menwe@amazon.com> * datasource annotation and import adjustment Signed-off-by: Eric <menwe@amazon.com> * remove pill effect Signed-off-by: Eric <menwe@amazon.com> * minor changes addressing latest comments Signed-off-by: Eric <menwe@amazon.com> * remove unused tests Signed-off-by: Eric <menwe@amazon.com> --------- Signed-off-by: Eric <menwe@amazon.com> Signed-off-by: Ashwin P Chandran <ashwinpc@amazon.com> Signed-off-by: Joshua Li <joshuali925@gmail.com> Co-authored-by: Josh Romero <rmerqg@amazon.com> Co-authored-by: Ashwin P Chandran <ashwinpc@amazon.com> Co-authored-by: Joshua Li <joshuali925@gmail.com> (cherry picked from commit 1e980fa) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Oct 6, 2023 · a5b3840 · a5b3840
1 parent 03bc3cc
commit a5b3840
Show file tree

Hide file tree

Showing 22 changed files with 1,099 additions and 79 deletions.
diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts
@@ -0,0 +1,79 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Abstract class representing a data source. This class provides foundational
+ * interfaces for specific data sources. Any data source connection needs to extend
+ * and implement from this base class
+ *
+ * DataSourceMetaData: Represents metadata associated with the data source.
+ * SourceDataSet: Represents the dataset associated with the data source.
+ * DataSourceQueryResult: Represents the result from querying the data source.
+ */
+
+import { ConnectionStatus } from './types';
+
+/**
+ * @experimental this class is experimental and might change in future releases.
+ */
+export abstract class DataSource<
+  DataSourceMetaData,
+  DataSetParams,
+  SourceDataSet,
+  DataSourceQueryParams,
+  DataSourceQueryResult
+> {
+  constructor(
+    private readonly name: string,
+    private readonly type: string,
+    private readonly metadata: DataSourceMetaData
+  ) {}
+
+  getName() {
+    return this.name;
+  }
+
+  getType() {
+    return this.type;
+  }
+
+  getMetadata() {
+    return this.metadata;
+  }
+
+  /**
+   * Abstract method to get the dataset associated with the data source.
+   * Implementing classes need to provide the specific implementation.
+   *
+   * Data source selector needs to display data sources with pattern
+   * group (connection name) - a list of datasets. For example, get
+   * all available tables for flint datasources, and get all index
+   * patterns for OpenSearch data source
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @returns {SourceDataSet} Dataset associated with the data source.
+   */
+  abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet;
+
+  /**
+   * Abstract method to run a query against the data source.
+   * Implementing classes need to provide the specific implementation.
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @returns {DataSourceQueryResult} Result from querying the data source.
+   */
+  abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult;
+
+  /**
+   * Abstract method to test the connection to the data source.
+   * Implementing classes should provide the specific logic to determine
+   * the connection status, typically indicating success or failure.
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @returns {ConnectionStatus | Promise<void>} Status of the connection test.
+   * @experimental
+   */
+  abstract testConnection(): ConnectionStatus | Promise<boolean>;
+}
diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts
@@ -0,0 +1,93 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { DataSourceFactory } from './factory';
+import { DataSource } from './datasource';
+import { IndexPattern, IndexPatternsService } from '../../index_patterns';
+
+class MockDataSource extends DataSource<any, any, any, any, any> {
+  private readonly indexPatterns;
+
+  constructor({
+    name,
+    type,
+    metadata,
+    indexPatterns,
+  }: {
+    name: string;
+    type: string;
+    metadata: any;
+    indexPatterns: IndexPatternsService;
+  }) {
+    super(name, type, metadata);
+    this.indexPatterns = indexPatterns;
+  }
+
+  async getDataSet(dataSetParams?: any) {
+    await this.indexPatterns.ensureDefaultIndexPattern();
+    return await this.indexPatterns.getCache();
+  }
+
+  async testConnection(): Promise<boolean> {
+    return true;
+  }
+
+  async runQuery(queryParams: any) {
+    return undefined;
+  }
+}
+
+describe('DataSourceFactory', () => {
+  beforeEach(() => {
+    // Reset the DataSourceFactory's singleton instance before each test for isolation
+    (DataSourceFactory as any).factory = undefined;
+  });
+
+  it('returns a singleton instance', () => {
+    const instance1 = DataSourceFactory.getInstance();
+    const instance2 = DataSourceFactory.getInstance();
+    expect(instance1).toBe(instance2);
+  });
+
+  it('registers a new data source type correctly', () => {
+    const factory = DataSourceFactory.getInstance();
+    expect(() => {
+      factory.registerDataSourceType('mock', MockDataSource);
+    }).not.toThrow();
+  });
+
+  it('throws error when registering an already registered data source type', () => {
+    const factory = DataSourceFactory.getInstance();
+    factory.registerDataSourceType('mock', MockDataSource);
+    expect(() => {
+      factory.registerDataSourceType('mock', MockDataSource);
+    }).toThrow('This data source type has already been registered');
+  });
+
+  it('creates and returns an instance of the registered data source type', () => {
+    const factory = DataSourceFactory.getInstance();
+    const mockIndexPattern = {} as IndexPattern;
+    const config = {
+      name: 'test_datasource',
+      type: 'mock',
+      metadata: null,
+      indexPattern: mockIndexPattern,
+    };
+    factory.registerDataSourceType('mock', MockDataSource);
+
+    const instance = factory.getDataSourceInstance('mock', config);
+    expect(instance).toBeInstanceOf(MockDataSource);
+    expect(instance.getName()).toEqual(config.name);
+    expect(instance.getType()).toEqual(config.type);
+    expect(instance.getMetadata()).toEqual(config.metadata);
+  });
+
+  it('throws error when trying to get an instance of an unregistered data source type', () => {
+    const factory = DataSourceFactory.getInstance();
+    expect(() => {
+      factory.getDataSourceInstance('unregistered', {});
+    }).toThrow('Unsupported data source type');
+  });
+});
diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts
@@ -0,0 +1,80 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * The DataSourceFactory is responsible for managing the registration and creation of data source classes.
+ * It serves as a registry for different data source types and provides a way to instantiate them.
+ */
+
+import { DataSourceType } from '../datasource_services';
+import { DataSource } from '../datasource';
+
+type DataSourceClass<
+  MetaData = any,
+  SetParams = any,
+  DataSet = any,
+  QueryParams = any,
+  QueryResult = any
+> = new (config: any) => DataSource<MetaData, SetParams, DataSet, QueryParams, QueryResult>;
+
+export class DataSourceFactory {
+  // Holds the singleton instance of the DataSourceFactory.
+  private static factory: DataSourceFactory;
+
+  // A dictionary holding the data source type as the key and its corresponding class constructor as the value.
+  private dataSourceClasses: { [type: string]: DataSourceClass } = {};
+
+  /**
+   * Private constructor to ensure only one instance of DataSourceFactory is created.
+   */
+  private constructor() {}
+
+  /**
+   * Returns the singleton instance of the DataSourceFactory. If it doesn't exist, it creates one.
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @returns {DataSourceFactory} The single instance of DataSourceFactory.
+   */
+  static getInstance(): DataSourceFactory {
+    if (!this.factory) {
+      this.factory = new DataSourceFactory();
+    }
+    return this.factory;
+  }
+
+  /**
+   * Registers a new data source type with its associated class.
+   * If the type has already been registered, an error is thrown.
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @param {string} type - The identifier for the data source type.
+   * @param {DataSourceClass} dataSourceClass - The constructor of the data source class.
+   * @throws {Error} Throws an error if the data source type has already been registered.
+   */
+  registerDataSourceType(type: string, dataSourceClass: DataSourceClass): void {
+    if (this.dataSourceClasses[type]) {
+      throw new Error('This data source type has already been registered');
+    }
+    this.dataSourceClasses[type] = dataSourceClass;
+  }
+
+  /**
+   * Creates and returns an instance of the specified data source type with the given configuration.
+   * If the type hasn't been registered, an error is thrown.
+   *
+   * @experimental This API is experimental and might change in future releases.
+   * @param {string} type - The identifier for the data source type.
+   * @param {any} config - The configuration for the data source instance.
+   * @returns {DataSourceType} An instance of the specified data source type.
+   * @throws {Error} Throws an error if the data source type is not supported.
+   */
+  getDataSourceInstance(type: string, config: any): DataSourceType {
+    const DataSourceClass = this.dataSourceClasses[type];
+    if (!DataSourceClass) {
+      throw new Error('Unsupported data source type');
+    }
+    return new DataSourceClass(config);
+  }
+}
diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export { DataSource } from './datasource';
+export {
+  IDataSourceMetaData,
+  ISourceDataSet,
+  IDataSetParams,
+  IDataSourceQueryParams,
+  IDataSourceQueryResult,
+  ConnectionStatus,
+  DataSourceConfig,
+  IndexPatternOption,
+} from './types';
+export { DataSourceFactory } from './factory';
diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts
@@ -0,0 +1,51 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @experimental These interfaces are experimental and might change in future releases.
+ */
+
+import { IndexPatternsService } from '../../index_patterns';
+import { DataSourceType } from '../datasource_services';
+
+export interface IndexPatternOption {
+  title: string;
+  id: string;
+}
+
+export interface IDataSourceMetaData {
+  name: string;
+}
+
+export interface IDataSourceGroup {
+  name: string;
+}
+
+export interface ISourceDataSet {
+  ds: DataSourceType;
+  data_sets: Array<string | IndexPatternOption>;
+}
+
+// to-dos: add common interfaces for datasource
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDataSetParams {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDataSourceQueryParams {}
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+export interface IDataSourceQueryResult {}
+
+export interface ConnectionStatus {
+  success: boolean;
+  info: string;
+}
+
+export interface DataSourceConfig {
+  name: string;
+  type: string;
+  metadata: any;
+  indexPatterns: IndexPatternsService;
+}