From aeb5fbe30301560a089950ea764f46eb97a7e6b7 Mon Sep 17 00:00:00 2001 From: Imri Paran Date: Tue, 13 Feb 2024 08:28:01 +0100 Subject: [PATCH] fixes #12591: add BigTable (#15122) * feat(connector): add BigTable * bigtable work 1. docstrings 2. tests 3. created a Row BaseModel 4. implemented a ClassConverter * docs moved to separate PR * format files * minor cosmetic - removed TODO - changed headers' year to 2024 for new files - fixed typos * format * formatting and comments 1. added missing docstrings. 2. abstracted the _find_instance method. 3. aliased the IDs used in the BigTable connection * added comment regarding private key * added comments regarding column families * enclose get_schema_name_list in `try/except/else` * format * streamlined get_schema_name_list to include all logic in the try block --- ingestion/setup.py | 1 + .../metadata/examples/workflows/bigtable.yaml | 32 ++ .../source/database/bigtable/__init__.py | 0 .../source/database/bigtable/client.py | 62 ++++ .../source/database/bigtable/connection.py | 116 +++++++ .../source/database/bigtable/metadata.py | 224 ++++++++++++++ .../source/database/bigtable/models.py | 60 ++++ .../source/database/common_nosql_source.py | 21 +- .../metadata/utils/datalake/datalake_utils.py | 8 +- .../unit/topology/database/test_bigtable.py | 290 ++++++++++++++++++ .../BigTableConnectionClassConverter.java | 37 +++ .../converter/ClassConverterFactory.java | 8 +- .../testConnections/database/bigtable.json | 29 ++ .../database/bigTableConnection.json | 51 +++ .../entity/services/databaseService.json | 7 + .../public/locales/en-US/Database/BigTable.md | 146 +++++++++ .../src/assets/img/service-icon-bigtable.png | Bin 0 -> 15715 bytes .../ui/src/constants/Services.constant.ts | 2 + .../ui/src/utils/DatabaseServiceUtils.ts | 6 + .../ui/src/utils/ServiceUtilClassBase.ts | 4 + 20 files changed, 1095 insertions(+), 9 deletions(-) create mode 100644 ingestion/src/metadata/examples/workflows/bigtable.yaml create mode 100644 ingestion/src/metadata/ingestion/source/database/bigtable/__init__.py create mode 100644 ingestion/src/metadata/ingestion/source/database/bigtable/client.py create mode 100644 ingestion/src/metadata/ingestion/source/database/bigtable/connection.py create mode 100644 ingestion/src/metadata/ingestion/source/database/bigtable/metadata.py create mode 100644 ingestion/src/metadata/ingestion/source/database/bigtable/models.py create mode 100644 ingestion/tests/unit/topology/database/test_bigtable.py create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/BigTableConnectionClassConverter.java create mode 100644 openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigTableConnection.json create mode 100644 openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigTable.md create mode 100644 openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-bigtable.png diff --git a/ingestion/setup.py b/ingestion/setup.py index 804f864e4e7e..f2dd6fb58d69 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -143,6 +143,7 @@ VERSIONS["pyarrow"], "sqlalchemy-bigquery>=1.2.2", }, + "bigtable": {"google-cloud-bigtable>=2.0.0", VERSIONS["pandas"]}, "clickhouse": {"clickhouse-driver~=0.2", "clickhouse-sqlalchemy~=0.2"}, "dagster": { VERSIONS["pymysql"], diff --git a/ingestion/src/metadata/examples/workflows/bigtable.yaml b/ingestion/src/metadata/examples/workflows/bigtable.yaml new file mode 100644 index 000000000000..652345d38bd1 --- /dev/null +++ b/ingestion/src/metadata/examples/workflows/bigtable.yaml @@ -0,0 +1,32 @@ +source: + type: bigtable + serviceName: local_bigtable + serviceConnection: + config: + type: BigTable + credentials: + gcpConfig: + type: service_account + projectId: project_id + privateKeyId: private_key_id + privateKey: private_key + clientEmail: gcpuser@project_id.iam.gserviceaccount.com + clientId: client_id + authUri: https://accounts.google.com/o/oauth2/auth + tokenUri: https://oauth2.googleapis.com/token + authProviderX509CertUrl: https://www.googleapis.com/oauth2/v1/certs + clientX509CertUrl: https://www.googleapis.com/oauth2/v1/certs + + sourceConfig: + config: + type: DatabaseMetadata +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/__init__.py b/ingestion/src/metadata/ingestion/source/database/bigtable/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/client.py b/ingestion/src/metadata/ingestion/source/database/bigtable/client.py new file mode 100644 index 000000000000..acb7e1883662 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/bigtable/client.py @@ -0,0 +1,62 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A client for Google Cloud Bigtable that supports multiple projects.""" +from functools import partial +from typing import List, Optional, Type + +from google import auth +from google.cloud.bigtable import Client + +NoProject = object() + + +class MultiProjectClient: + """Google Cloud Client does not support ad-hoc project switching. This class wraps the client and allows + switching between projects. If no project is specified, the client will not have a project set and will try + to resolve it from ADC. + Example usage: + ``` + from google.cloud.bigtable import Client + client = MultiProjectClient(Client, project_ids=["project1", "project2"]) + instances_project1 = client.list_instances("project1") + instances_project2 = client.list_instances("project2") + """ + + def __init__( + self, + client_class: Type[Client], + project_ids: Optional[List[str]] = None, + **client_kwargs, + ): + if project_ids: + self.clients = { + project_id: client_class(project=project_id, **client_kwargs) + for project_id in project_ids + } + else: + self.clients = {NoProject: client_class(**client_kwargs)} + + def project_ids(self): + if NoProject in self.clients: + _, project_id = auth.default() + return [project_id] + return list(self.clients.keys()) + + def __getattr__(self, client_method): + """Return the underlying client method as a partial function so we can inject the project_id.""" + return partial(self._call, client_method) + + def _call(self, method, project_id, *args, **kwargs): + """Call the method on the client for the given project_id. The args and kwargs are passed through.""" + client = self.clients.get(project_id, self.clients.get(NoProject)) + if not client: + raise ValueError(f"Project {project_id} not found") + return getattr(client, method)(*args, **kwargs) diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py new file mode 100644 index 000000000000..754424d53270 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py @@ -0,0 +1,116 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BigTable connection""" +from typing import List, Optional + +from google.cloud.bigtable import Client + +from metadata.generated.schema.entity.automations.workflow import ( + Workflow as AutomationWorkflow, +) +from metadata.generated.schema.entity.services.connections.database.bigTableConnection import ( + BigTableConnection, +) +from metadata.generated.schema.security.credentials.gcpValues import ( + GcpCredentialsValues, + SingleProjectId, +) +from metadata.ingestion.connections.test_connections import ( + SourceConnectionException, + test_connection_steps, +) +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.bigtable.client import MultiProjectClient +from metadata.utils.credentials import set_google_credentials +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + + +def get_connection(connection: BigTableConnection): + set_google_credentials(connection.credentials) + project_ids = None + if isinstance(connection.credentials.gcpConfig, GcpCredentialsValues): + project_ids = ( + [connection.credentials.gcpConfig.projectId.__root__] + if isinstance(connection.credentials.gcpConfig.projectId, SingleProjectId) + else connection.credentials.gcpConfig.projectId.__root__ + ) + # admin=True is required to list instances and tables + return MultiProjectClient(client_class=Client, project_ids=project_ids, admin=True) + + +def get_nested_index(lst: list, index: List[int], default=None): + try: + for i in index: + lst = lst[i] + return lst + except IndexError: + return default + + +class Tester: + """ + A wrapper class that holds state. We need it because the different testing stages + are not independent of each other. For example, we need to list instances before we can list + """ + + def __init__(self, client: MultiProjectClient): + self.client = client + self.project_id = None + self.instance = None + self.table = None + + def list_instances(self): + self.project_id = list(self.client.clients.keys())[0] + instances = list(self.client.list_instances(project_id=self.project_id)) + self.instance = get_nested_index(instances, [0, 0]) + + def list_tables(self): + if not self.instance: + raise SourceConnectionException( + f"No instances found in project {self.project_id}" + ) + tables = list(self.instance.list_tables()) + self.table = tables[0] + + def get_row(self): + if not self.table: + raise SourceConnectionException( + f"No tables found in project {self.instance.project_id} and instance {self.instance.instance_id}" + ) + self.table.read_rows(limit=1) + + +def test_connection( + metadata: OpenMetadata, + client: MultiProjectClient, + service_connection: BigTableConnection, + automation_workflow: Optional[AutomationWorkflow] = None, +) -> None: + """ + Test connection. This can be executed either as part + of a metadata workflow or during an Automation Workflow + """ + tester = Tester(client) + + test_fn = { + "GetInstances": tester.list_instances, + "GetTables": tester.list_tables, + "GetRows": tester.get_row, + } + + test_connection_steps( + metadata=metadata, + test_fn=test_fn, + service_type=service_connection.type.value, + automation_workflow=automation_workflow, + ) diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/metadata.py b/ingestion/src/metadata/ingestion/source/database/bigtable/metadata.py new file mode 100644 index 000000000000..a2a072db53f0 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/bigtable/metadata.py @@ -0,0 +1,224 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Bigtable source methods. +""" +import traceback +from typing import Dict, Iterable, List, Optional, Union + +from google.cloud.bigtable import row_filters +from google.cloud.bigtable.instance import Instance +from google.cloud.bigtable.table import Table + +from metadata.generated.schema.entity.data.table import ( + ConstraintType, + TableConstraint, + TableType, +) +from metadata.generated.schema.entity.services.connections.database.bigTableConnection import ( + BigTableConnection, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) +from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.bigtable.client import MultiProjectClient +from metadata.ingestion.source.database.bigtable.models import Row +from metadata.ingestion.source.database.common_nosql_source import ( + SAMPLE_SIZE as GLOBAL_SAMPLE_SIZE, +) +from metadata.ingestion.source.database.common_nosql_source import CommonNoSQLSource +from metadata.ingestion.source.database.multi_db_source import MultiDBSource +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + +# BigTable group's its columns in column families. We make an assumption that if the table has a big number of +# columns, we at least get a sample of the first 100 column families. +MAX_COLUMN_FAMILIES = 100 +SAMPLES_PER_COLUMN_FAMILY = 100 + +ProjectId = str +InstanceId = str +TableId = str + + +class BigtableSource(CommonNoSQLSource, MultiDBSource): + """ + Implements the necessary methods to extract database metadata from Google BigTable Source. + BigTable is a NoSQL database service for handling large amounts of data. Tha mapping is as follows: + project -> instance -> table -> column_family.column + (database) (schema) + For more infor about BigTable: https://cloud.google.com/bigtable/?hl=en + All data types are registered as bytes. + """ + + def __init__(self, config: WorkflowSource, metadata: OpenMetadata): + super().__init__(config, metadata) + self.client: MultiProjectClient = self.connection_obj + + # ths instances and tables are cached to avoid making redundant requests to the API. + self.instances: Dict[ProjectId, Dict[InstanceId, Instance]] = {} + self.tables: Dict[ProjectId, Dict[InstanceId, Dict[TableId, Table]]] = {} + + @classmethod + def create(cls, config_dict, metadata: OpenMetadata): + config: WorkflowSource = WorkflowSource.parse_obj(config_dict) + connection: BigTableConnection = config.serviceConnection.__root__.config + if not isinstance(connection, BigTableConnection): + raise InvalidSourceException( + f"Expected BigTableConnection, but got {connection}" + ) + return cls(config, metadata) + + def get_configured_database(self) -> Optional[str]: + """ + This connector uses "virtual databases" in the form of GCP projects. + The concept of a default project for the GCP client is not useful here because the project ID + is always an explicit part of the connection. Therefore, this method returns None and the databases + are resolved using `self.get_database_names`. + """ + return None + + def get_database_names(self) -> Iterable[str]: + return self.get_database_names_raw() + + def get_database_names_raw(self) -> Iterable[str]: + yield from self.client.project_ids() + + def get_schema_name_list(self) -> List[str]: + project_id = self.context.database + try: + # the first element is a list of instances + # the second element is another collection (seems empty) and I do not know what is its purpose + instances, _ = self.client.list_instances(project_id=project_id) + self.instances[project_id] = { + instance.instance_id: instance for instance in instances + } + return list(self.instances[project_id].keys()) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.error( + f"Failed to list BigTable instances in project {project_id}: {err}" + ) + raise + + def get_table_name_list(self, schema_name: str) -> List[str]: + project_id = self.context.database + try: + instance = self._get_instance(project_id, schema_name) + if instance is None: + raise RuntimeError(f"Instance {project_id}/{schema_name} not found.") + tables = instance.list_tables() + for table in tables: + self._set_nested( + self.tables, + [project_id, instance.instance_id, table.table_id], + table, + ) + return list(self.tables[project_id][schema_name].keys()) + except Exception as err: + logger.debug(traceback.format_exc()) + # add context to the error message + logger.error( + f"Failed to list BigTable table names in {project_id}.{schema_name}: {err}" + ) + return [] + + def get_table_constraints( + self, db_name: str, schema_name: str, table_name: str + ) -> List[TableConstraint]: + return [ + TableConstraint( + constraintType=ConstraintType.PRIMARY_KEY, columns=["row_key"] + ) + ] + + def get_table_columns_dict( + self, schema_name: str, table_name: str + ) -> Union[List[Dict], Dict]: + project_id = self.context.database + try: + table = self._get_table(project_id, schema_name, table_name) + if table is None: + raise RuntimeError( + f"Table {project_id}/{schema_name}/{table_name} not found." + ) + column_families = table.list_column_families() + # all BigTable tables have a "row_key" column. Even if there are no records in the table. + records = [{"row_key": b"row_key"}] + # In order to get a "good" sample of data, we try to distribute the sampling + # across multiple column families. + for cf in list(column_families.keys())[:MAX_COLUMN_FAMILIES]: + records.extend( + self._get_records_for_column_family( + table, cf, SAMPLES_PER_COLUMN_FAMILY + ) + ) + if len(records) >= GLOBAL_SAMPLE_SIZE: + break + return records + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to read BigTable rows for [{project_id}.{schema_name}.{table_name}]: {err}" + ) + return [] + + def get_source_url( + self, + database_name: Optional[str] = None, + schema_name: Optional[str] = None, + table_name: Optional[str] = None, + table_type: Optional[TableType] = None, + ) -> Optional[str]: + """ + Method to get the source url for a BigTable table + """ + try: + if schema_name and table_name: + return ( + "https://console.cloud.google.com/bigtable/instances/" + f"{schema_name}/tables/{table_name}/overview?project={database_name}" + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error(f"Unable to get source url: {exc}") + return None + + @staticmethod + def _set_nested(dct: dict, keys: List[str], value: any) -> None: + for key in keys[:-1]: + dct = dct.setdefault(key, {}) + dct[keys[-1]] = value + + @staticmethod + def _get_records_for_column_family( + table: Table, column_family: str, limit: int + ) -> List[Dict]: + filter_ = row_filters.ColumnRangeFilter(column_family_id=column_family) + rows = table.read_rows(limit=limit, filter_=filter_) + return [Row.from_partial_row(row).to_record() for row in rows] + + def _get_table( + self, project_id: str, schema_name: str, table_name: str + ) -> Optional[Table]: + try: + return self.tables[project_id][schema_name][table_name] + except KeyError: + return None + + def _get_instance(self, project_id: str, schema_name: str) -> Optional[Instance]: + try: + return self.instances[project_id][schema_name] + except KeyError: + return None diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/models.py b/ingestion/src/metadata/ingestion/source/database/bigtable/models.py new file mode 100644 index 000000000000..f8da387c8a55 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/database/bigtable/models.py @@ -0,0 +1,60 @@ +# Copyright 2024 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Bigtable source models. +""" +from typing import Dict, List + +from google.cloud.bigtable.row import PartialRowData +from pydantic import BaseModel + + +class Value(BaseModel): + """A Bigtable cell value.""" + + timestamp: int + value: bytes + + +class Cell(BaseModel): + """A Bigtable cell.""" + + values: List[Value] + + +class Row(BaseModel): + """A Bigtable row.""" + + cells: Dict[str, Dict[bytes, Cell]] + row_key: bytes + + @classmethod + def from_partial_row(cls, row: PartialRowData): + cells = {} + for cf, cf_cells in row.cells.items(): + cells.setdefault(cf, {}) + for column, cell in cf_cells.items(): + cells[cf][column] = Cell( + values=[Value(timestamp=c.timestamp, value=c.value) for c in cell] + ) + return cls(cells=cells, row_key=row.row_key) + + def to_record(self) -> Dict[str, bytes]: + record = {} + for cf, cells in self.cells.items(): + for column, cell in cells.items(): + # Since each cell can have multiple values and the API returns them in descending order + # from latest to oldest, we only take the latest value. This probably does not matter since + # all we care about is data types and all data stored in BigTable is of type `bytes`. + record[f"{cf}.{column.decode()}"] = cell.values[0].value + record["row_key"] = self.row_key + + return record diff --git a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py index e9f394243ff2..8956abb27ddc 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py @@ -28,7 +28,11 @@ from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema -from metadata.generated.schema.entity.data.table import Table, TableType +from metadata.generated.schema.entity.data.table import ( + Table, + TableConstraint, + TableType, +) from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -203,6 +207,15 @@ def get_table_columns_dict( need to be overridden by sources """ + def get_table_constraints( + self, + db_name: str, + schema_name: str, + table_name: str, + ) -> Optional[List[TableConstraint]]: + # pylint: disable=unused-argument + return None + def yield_table( self, table_name_and_type: Tuple[str, str] ) -> Iterable[Either[CreateTableRequest]]: @@ -223,7 +236,11 @@ def yield_table( name=table_name, tableType=table_type, columns=columns, - tableConstraints=None, + tableConstraints=self.get_table_constraints( + schema_name=schema_name, + table_name=table_name, + db_name=self.context.database, + ), databaseSchema=fqn.build( metadata=self.metadata, entity_type=DatabaseSchema, diff --git a/ingestion/src/metadata/utils/datalake/datalake_utils.py b/ingestion/src/metadata/utils/datalake/datalake_utils.py index 369b1e3e0e21..e067443090f9 100644 --- a/ingestion/src/metadata/utils/datalake/datalake_utils.py +++ b/ingestion/src/metadata/utils/datalake/datalake_utils.py @@ -169,6 +169,7 @@ class GenericDataFrameColumnParser: ["datetime64", "timedelta[ns]", "datetime64[ns]"], DataType.DATETIME ), "str": DataType.STRING, + "bytes": DataType.BYTES, } def __init__(self, data_frame: "DataFrame"): @@ -247,8 +248,13 @@ def fetch_col_types(cls, data_frame, column_name): data_type = "string" data_type = cls._data_formats.get( - data_type or data_frame[column_name].dtypes.name, DataType.STRING + data_type or data_frame[column_name].dtypes.name, ) + if not data_type: + logger.debug( + f"unknown data type {data_frame[column_name].dtypes.name}. resolving to string." + ) + data_type = data_type or DataType.STRING except Exception as err: logger.warning( f"Failed to distinguish data type for column {column_name}, Falling back to {data_type}, exc: {err}" diff --git a/ingestion/tests/unit/topology/database/test_bigtable.py b/ingestion/tests/unit/topology/database/test_bigtable.py new file mode 100644 index 000000000000..6ca32b5f8475 --- /dev/null +++ b/ingestion/tests/unit/topology/database/test_bigtable.py @@ -0,0 +1,290 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test MongoDB using the topology +""" + +import json +from pathlib import Path +from unittest import TestCase +from unittest.mock import Mock, patch + +import pytest + +from metadata.generated.schema.api.data.createTable import CreateTableRequest +from metadata.generated.schema.entity.data.database import Database +from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema +from metadata.generated.schema.entity.data.table import ( + Column, + ConstraintType, + DataType, + TableConstraint, + TableType, +) +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, + DatabaseServiceType, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.basic import SourceUrl +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.database.bigtable.metadata import BigtableSource + +mock_file_path = ( + Path(__file__).parent.parent.parent / "resources/datasets/glue_db_dataset.json" +) +with open(mock_file_path) as file: + mock_data: dict = json.load(file) + +mock_bigtable_config = { + "source": { + "type": "bigtable", + "serviceName": "local_bigtable", + "serviceConnection": { + "config": { + "type": "BigTable", + "credentials": { + "gcpConfig": { + "type": "service_account", + "projectId": "my-gcp-project", + "privateKeyId": "private_key_id", + # this is a valid key that was generated on a local machine and is not used for any real project + "privateKey": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAw3vHG9fDIkcYB0xi2Mv4fS2gUzKR9ZRrcVNeKkqGFTT71AVB\nOzgIqYVe8b2aWODuNye6sipcrqTqOt05Esj+sxhk5McM9bE2RlxXC5QH/Bp9zxMP\n/Yksv9Ov7fdDt/loUk7sTXvI+7LDJfmRYU6MtVjyyLs7KpQIB2xBWEToU1xZY+v0\ndRC1NA+YWc+FjXbAiFAf9d4gXkYO8VmU5meixVh4C8nsjokEXk0T/HEItpZCxadk\ndZ7LKUE/HDmWCO2oNG6sCf4ET2crjSdYIfXuREopX1aQwnk7KbI4/YIdlRz1I369\nAz3+Hxlf9lLJVH3+itN4GXrR9yWWKWKDnwDPbQIDAQABAoIBAQC3X5QuTR7SN8iV\niBUtc2D84+ECSmza5shG/UJW/6N5n0Mf53ICgBS4GNEwiYCRISa0/ILIgK6CcVb7\nsuvH8F3kWNzEMui4TO0x4YsR5GH9HkioCCS224frxkLBQnL20HIIy9ok8Rpe6Zjg\nNZUnp4yczPyqSeA9l7FUbTt69uDM2Cx61m8REOpFukpnYLyZGbmNPYmikEO+rq9r\nwNID5dkSeVuQYo4MQdRavOGFUWvUYXzkEQ0A6vPyraVBfolESX8WaLNVjic7nIa3\nujdSNojnJqGJ3gslntcmN1d4JOfydc4bja4/NdNlcOHpWDGLzY1QnaDe0Koxn8sx\nLT9MVD2NAoGBAPy7r726bKVGWcwqTzUuq1OWh5c9CAc4N2zWBBldSJyUdllUq52L\nWTyva6GRoRzCcYa/dKLLSM/k4eLf9tpxeIIfTOMsvzGtbAdm257ndMXNvfYpxCfU\nK/gUFfAUGHZ3MucTHRY6DTkJg763Sf6PubA2fqv3HhVZDK/1HGDtHlTPAoGBAMYC\npdV7O7lAyXS/d9X4PQZ4BM+P8MbXEdGBbPPlzJ2YIb53TEmYfSj3z41u9+BNnhGP\n4uzUyAR/E4sxrA2+Ll1lPSCn+KY14WWiVGfWmC5j1ftdpkbrXstLN8NpNYzrKZwx\njdR0ZkwvZ8B5+kJ1hK96giwWS+SJxJR3TohcQ18DAoGAJSfmv2r//BBqtURnHrd8\nwq43wvlbC8ytAVg5hA0d1r9Q4vM6w8+vz+cuWLOTTyobDKdrG1/tlXrd5r/sh9L0\n15SIdkGm3kPTxQbPNP5sQYRs8BrV1tEvoao6S3B45DnEBwrdVN42AXOvpcNGoqE4\nuHpahyeuiY7s+ZV8lZdmxSsCgYEAolr5bpmk1rjwdfGoaKEqKGuwRiBX5DHkQkxE\n8Zayt2VOBcX7nzyRI05NuEIMrLX3rZ61CktN1aH8fF02He6aRaoE/Qm9L0tujM8V\nNi8WiLMDeR/Ifs3u4/HAv1E8v1byv0dCa7klR8J257McJ/ID4X4pzcxaXgE4ViOd\nGOHNu9ECgYEApq1zkZthEQymTUxs+lSFcubQpaXyf5ZC61cJewpWkqGDtSC+8DxE\nF/jydybWuoNHXymnvY6QywxuIooivbuib6AlgpEJeybmnWlDOZklFOD0abNZ+aNO\ndUk7XVGffCakXQ0jp1kmZA4lGsYK1h5dEU5DgXqu4UYJ88Vttax2W+Y=\n-----END RSA PRIVATE KEY-----\n", + "clientEmail": "gcpuser@project_id.iam.gserviceaccount.com", + "clientId": "client_id", + "authUri": "https://accounts.google.com/o/oauth2/auth", + "tokenUri": "https://oauth2.googleapis.com/token", + "authProviderX509CertUrl": "https://www.googleapis.com/oauth2/v1/certs", + "clientX509CertUrl": "https://www.googleapis.com/oauth2/v1/certs", + } + }, + }, + }, + "sourceConfig": { + "config": { + "type": "DatabaseMetadata", + "schemaFilterPattern": {"includes": ["my_instance"]}, + "tableFilterPattern": {"includes": ["random_table"]}, + } + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + } + }, +} + +MOCK_DATABASE_SERVICE = DatabaseService( + id="85811038-099a-11ed-861d-0242ac120002", + name="local_bigtable", + connection=DatabaseConnection(), + serviceType=DatabaseServiceType.Glue, +) + +MOCK_DATABASE = Database( + id="2aaa012e-099a-11ed-861d-0242ac120002", + name="my-gcp-project", + fullyQualifiedName="local_bigtable.my-gcp-project", + displayName="my-gcp-project", + description="", + service=EntityReference( + id="85811038-099a-11ed-861d-0242ac120002", + type="databaseService", + ), +) + +MOCK_DATABASE_SCHEMA = DatabaseSchema( + id="2aaa012e-099a-11ed-861d-0242ac120056", + name="my_instance", + fullyQualifiedName="local_bigtable.my-gcp-project.my_instance", + displayName="default", + description="", + database=EntityReference( + id="2aaa012e-099a-11ed-861d-0242ac120002", + type="database", + ), + service=EntityReference( + id="85811038-099a-11ed-861d-0242ac120002", + type="databaseService", + ), +) + + +MOCK_CREATE_TABLE = CreateTableRequest( + name="random_table", + tableType=TableType.Regular, + columns=[ + Column( + name="row_key", + displayName="row_key", + dataType=DataType.BYTES, + dataTypeDisplay=DataType.BYTES.value, + ), + Column( + name="cf1.col1", + displayName="cf1.col1", + dataType=DataType.BYTES, + dataTypeDisplay=DataType.BYTES.value, + ), + Column( + name="cf2.col2", + displayName="cf2.col2", + dataType=DataType.BYTES, + dataTypeDisplay=DataType.BYTES.value, + ), + ], + tableConstraints=[ + TableConstraint(constraintType=ConstraintType.PRIMARY_KEY, columns=["row_key"]) + ], + databaseSchema="local_bigtable.my-gcp-project.my_instance", + sourceUrl=SourceUrl( + __root__="https://console.cloud.google.com/bigtable/instances/my_instance/tables/random_table/overview?project=my-gcp-project" + ), +) + + +EXPECTED_DATABASE_NAMES = ["my-gcp-project"] + +EXPECTED_DATABASE_SCHEMA_NAMES = [ + "my_instance", +] + +MOCK_DATABASE_SCHEMA_NAMES = [ + "my_instance", + "random1_schema", +] + +EXPECTED_TABLE_NAMES = [ + ("random_table", TableType.Regular), +] + + +def custom_column_compare(self, other): + return ( + self.name == other.name + and self.description == other.description + and self.children == other.children + ) + + +@pytest.fixture +def mock_bigtable_row(): + mock = Mock() + cell = Mock() + cell.value = b"cell_value" + cell.timestamp = 1234567890 + mock.cells = {"cf1": {b"col1": [cell]}, "cf2": {b"col2": [cell]}} + mock.row_key = b"row_key" + yield mock + + +@pytest.fixture +def mock_bigtable_table(mock_bigtable_row): + mock = Mock() + mock.table_id = "random_table" + mock.list_column_families.return_value = {"cf1": None, "cf2": None} + mock.read_rows.return_value = [mock_bigtable_row] + yield mock + + +@pytest.fixture +def mock_bigtable_instance(mock_bigtable_table): + mock = Mock() + mock.instance_id = "my_instance" + mock.project_id = "my-gcp-project" + mock.list_tables.return_value = [mock_bigtable_table] + yield mock + + +@pytest.fixture +def mock_google_cloud_client(mock_bigtable_instance): + with patch("google.cloud.bigtable.Client") as mock_client: + mock_client.list_instances.return_value = [[], []] + mock_client().list_instances.return_value = [[mock_bigtable_instance], []] + yield mock_client + + +@pytest.fixture +def mock_test_connection(): + with patch.object(BigtableSource, "test_connection") as mock_test_connection: + mock_test_connection.return_value = True + yield mock_test_connection + + +class BigTableUnitTest(TestCase): + @pytest.fixture(autouse=True) + def setup( + self, + monkeypatch, + mock_google_cloud_client, + mock_test_connection, + mock_bigtable_instance, + mock_bigtable_table, + ): + self.config = OpenMetadataWorkflowConfig.parse_obj(mock_bigtable_config) + self.bigtable_source = BigtableSource.create( + mock_bigtable_config["source"], + OpenMetadata(self.config.workflowConfig.openMetadataServerConfig), + ) + self.bigtable_source.context.__dict__[ + "database_service" + ] = MOCK_DATABASE_SERVICE.name.__root__ + self.bigtable_source.context.__dict__["database"] = MOCK_DATABASE.name.__root__ + self.bigtable_source.context.__dict__[ + "database_schema" + ] = MOCK_DATABASE_SCHEMA.name.__root__ + self.bigtable_source.instances = { + "my-gcp-project": { + mock_bigtable_instance.instance_id: mock_bigtable_instance + } + } + self.bigtable_source.tables = { + "my-gcp-project": { + mock_bigtable_instance.instance_id: { + mock_bigtable_table.table_id: mock_bigtable_table + } + } + } + + def test_database_names(self): + assert ( + list(self.bigtable_source.get_database_names()) == EXPECTED_DATABASE_NAMES + ) + + def test_database_schema_names(self): + assert ( + list(self.bigtable_source.get_database_schema_names()) + == EXPECTED_DATABASE_SCHEMA_NAMES + ) + + def test_table_names(self): + assert ( + list(self.bigtable_source.get_tables_name_and_type()) + == EXPECTED_TABLE_NAMES + ) + + def test_yield_tables(self): + Column.__eq__ = custom_column_compare + result = next(self.bigtable_source.yield_table(EXPECTED_TABLE_NAMES[0])) + assert result.left is None + assert result.right.name.__root__ == "random_table" + assert result.right == MOCK_CREATE_TABLE diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/BigTableConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/BigTableConnectionClassConverter.java new file mode 100644 index 000000000000..b26383645d1b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/BigTableConnectionClassConverter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.secrets.converter; + +import java.util.List; +import org.openmetadata.schema.security.credentials.GCPCredentials; +import org.openmetadata.schema.services.connections.database.BigTableConnection; +import org.openmetadata.service.util.JsonUtils; + +/** Converter class to get an `BigTableConnection` object. */ +public class BigTableConnectionClassConverter extends ClassConverter { + + public BigTableConnectionClassConverter() { + super(BigTableConnection.class); + } + + @Override + public Object convert(Object object) { + BigTableConnection connection = (BigTableConnection) JsonUtils.convertValue(object, this.clazz); + + tryToConvertOrFail(connection.getCredentials(), List.of(GCPCredentials.class)) + .ifPresent(obj -> connection.setCredentials((GCPCredentials) obj)); + + return connection; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java index c428b4b5acad..0da02d5a894d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java @@ -24,12 +24,7 @@ import org.openmetadata.schema.services.connections.dashboard.LookerConnection; import org.openmetadata.schema.services.connections.dashboard.SupersetConnection; import org.openmetadata.schema.services.connections.dashboard.TableauConnection; -import org.openmetadata.schema.services.connections.database.BigQueryConnection; -import org.openmetadata.schema.services.connections.database.DatalakeConnection; -import org.openmetadata.schema.services.connections.database.IcebergConnection; -import org.openmetadata.schema.services.connections.database.MysqlConnection; -import org.openmetadata.schema.services.connections.database.PostgresConnection; -import org.openmetadata.schema.services.connections.database.TrinoConnection; +import org.openmetadata.schema.services.connections.database.*; import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.database.iceberg.IcebergFileSystem; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; @@ -49,6 +44,7 @@ private ClassConverterFactory() { Map.ofEntries( Map.entry(AirflowConnection.class, new AirflowConnectionClassConverter()), Map.entry(BigQueryConnection.class, new BigQueryConnectionClassConverter()), + Map.entry(BigTableConnection.class, new BigTableConnectionClassConverter()), Map.entry(DatalakeConnection.class, new DatalakeConnectionClassConverter()), Map.entry(MysqlConnection.class, new MysqlConnectionClassConverter()), Map.entry(TrinoConnection.class, new TrinoConnectionClassConverter()), diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json b/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json new file mode 100644 index 000000000000..86ca0bb17ac2 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/testConnections/database/bigtable.json @@ -0,0 +1,29 @@ +{ + "name": "BigTable", + "displayName": "BigTable Test Connection", + "description": "This Test Connection validates the access against the database and basic metadata extraction of schemas and tables.", + "steps": [ + { + "name": "GetInstances", + "description": "Validate that we can get the instances with the given credentials.", + "errorMessage": "Failed to get BigTable instances, please validate to the credentials of service account", + "shortCircuit": true, + "mandatory": true + }, + { + "name": "GetTables", + "description": "Validate that we can get tables with the given credentials.", + "errorMessage": "Failed to get BigTable tables, please validate to the credentials of service account", + "shortCircuit": true, + "mandatory": true + }, + { + "name": "ReadRows", + "description": "Validate that we can read rows with the given credentials.", + "errorMessage": "Failed to read rows from BigTable, please validate to the credentials of service account" + "shortCircuit": true, + "mandatory": true + } + ] + } + diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigTableConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigTableConnection.json new file mode 100644 index 000000000000..513c04232a47 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigTableConnection.json @@ -0,0 +1,51 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/database/bigTableConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BigTableConnection", + "description": "Google BigTable Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.database.BigTableConnection", + "definitions": { + "bigtableType": { + "description": "Service type.", + "type": "string", + "enum": [ + "BigTable" + ], + "default": "BigTable" + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/bigtableType", + "default": "BigTable" + }, + "credentials": { + "title": "GCP Credentials", + "description": "GCP Credentials", + "$ref": "../../../../security/credentials/gcpCredentials.json" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" + }, + "connectionArguments": { + "title": "Connection Arguments", + "$ref": "../connectionBasicType.json#/definitions/connectionArguments" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + }, + "supportsDatabase": { + "title": "Supports Database", + "$ref": "../connectionBasicType.json#/definitions/supportsDatabase" + } + }, + "additionalProperties": false, + "required": [ + "credentials" + ] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json index 9b87a31c033b..b03d412e02fb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/databaseService.json @@ -16,6 +16,7 @@ "type": "string", "enum": [ "BigQuery", + "BigTable", "Mysql", "Redshift", "Snowflake", @@ -59,6 +60,9 @@ { "name": "BigQuery" }, + { + "name": "BigTable" + }, { "name": "Mysql" }, @@ -188,6 +192,9 @@ { "$ref": "./connections/database/bigQueryConnection.json" }, + { + "$ref": "./connections/database/bigTableConnection.json" + }, { "$ref": "./connections/database/athenaConnection.json" }, diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigTable.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigTable.md new file mode 100644 index 000000000000..3dd8b696f0ce --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigTable.md @@ -0,0 +1,146 @@ +# BigTable + +In this section, we provide guides and references to use the BigTable connector. + +## Requirements + +We need to enable the BigTable API and use an account with a specific set of minimum permissions: + +### BigTable API Permissions + +Click on `Enable API`, which will enable the APs on the selected project: + +- [Cloud Bigtable Admin API ](https://console.cloud.google.com/apis/api/bigtableadmin.googleapis.com) +- [Cloud Bigtable API](https://console.cloud.google.com/apis/library/bigtable.googleapis.com) + +### GCP Permissions + +To execute the metadata extraction and Usage workflow successfully, the user or the service account should have enough permissions to fetch required data: + +- `bigtable.instances.get` +- `bigtable.instances.list` +- `bigtable.tables.get` +- `bigtable.tables.list` +- `bigtable.tables.readRows` + +You can visit [this](https://docs.open-metadata.org/connectors/database/bigtable/roles) documentation on how you can create a custom role in GCP and assign the above permissions to the role & service account! + +You can find further information on the BigTable connector in the [docs](https://docs.open-metadata.org/connectors/database/bigtable). + +## Connection Details + +$$section +### Scheme $(id="scheme") + +SQLAlchemy driver scheme options. +$$ + +$$section +### Host Port $(id="hostPort") + +BigQuery APIs URL. By default, the API URL is `bigquery.googleapis.com`. You can modify this if you have custom implementation of BigQuery. +$$ + +$$section +### GCP Credentials Configuration $(id="gcpConfig") + +You can authenticate with your BigQuery instance using either `GCP Credentials Path` where you can specify the file path of the service account key, or you can pass the values directly by choosing the `GCP Credentials Values` from the service account key file. + +You can check [this](https://cloud.google.com/iam/docs/keys-create-delete#iam-service-account-keys-create-console) documentation on how to create the service account keys and download it. + +If you want to use [ADC authentication](https://cloud.google.com/docs/authentication#adc) for BigQuery you can just leave the GCP credentials empty. + +$$ + +$$section +### Credentials Type $(id="type") + +Credentials Type is the type of the account, for a service account the value of this field is `service_account`. To fetch this key, look for the value associated with the `type` key in the service account key file. +$$ + +$$section +### Project ID $(id="projectId") + +A project ID is a unique string used to differentiate your project from all others in Google Cloud. To fetch this key, look for the value associated with the `project_id` key in the service account key file. +$$ + +$$section +### Private Key ID $(id="privateKeyId") + +This is a unique identifier for the private key associated with the service account. To fetch this key, look for the value associated with the `private_key_id` key in the service account file. +$$ + +$$section +### Private Key $(id="privateKey") + +This is the private key associated with the service account that is used to authenticate and authorize access to GCP. To fetch this key, look for the value associated with the `private_key` key in the service account file. + +Make sure you are passing the key in a correct format. If your private key looks like this: + +``` +-----BEGIN ENCRYPTED PRIVATE KEY----- +MII.. +MBQ... +CgU.. +8Lt.. +... +h+4= +-----END ENCRYPTED PRIVATE KEY----- +``` + +You will have to replace new lines with `\n` and the final private key that you need to pass should look like this: + +``` +-----BEGIN ENCRYPTED PRIVATE KEY-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END ENCRYPTED PRIVATE KEY-----\n +``` +$$ + +$$section +### Client Email $(id="clientEmail") + +This is the email address associated with the service account. To fetch this key, look for the value associated with the `client_email` key in the service account key file. +$$ + +$$section +### Client ID $(id="clientId") + +This is a unique identifier for the service account. To fetch this key, look for the value associated with the `client_id` key in the service account key file. +$$ + +$$section +### Auth URI $(id="authUri") + +This is the URI for the authorization server. To fetch this key, look for the value associated with the `auth_uri` key in the service account key file. +$$ + +$$section +### Token URI $(id="tokenUri") + +The Google Cloud Token URI is a specific endpoint used to obtain an OAuth 2.0 access token from the Google Cloud IAM service. This token allows you to authenticate and access various Google Cloud resources and APIs that require authorization. + +To fetch this key, look for the value associated with the `token_uri` key in the service account credentials file. +$$ + +$$section +### Auth Provider X509Cert URL $(id="authProviderX509CertUrl") + +This is the URL of the certificate that verifies the authenticity of the authorization server. To fetch this key, look for the value associated with the `auth_provider_x509_cert_url` key in the service account key file. +$$ + +$$section +### Client X509Cert URL $(id="clientX509CertUrl") + +This is the URL of the certificate that verifies the authenticity of the service account. To fetch this key, look for the value associated with the `client_x509_cert_url` key in the service account key file. +$$ + +$$section +### Connection Options $(id="connectionOptions") + +Additional connection options to build the URL that can be sent to service during the connection. +$$ + +$$section +### Connection Arguments $(id="connectionArguments") + +Additional connection arguments such as security or protocol configs that can be sent to service during connection. +$$ diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-bigtable.png b/openmetadata-ui/src/main/resources/ui/src/assets/img/service-icon-bigtable.png new file mode 100644 index 0000000000000000000000000000000000000000..19f9da21e847f4d758fde8c6db07e280ed898883 GIT binary patch literal 15715 zcmaJ|V{{~4w2f_>9ZhW8wlQ%su|2V^i8*m5HYeu9wrx)`(W~$0`|;lD)m7cqt4>v) zd+y%n>~kwhMM)YNfdByv3=COTMnVnPkN)?Dg8{bHIyHH~4$4YYQ4|cUJ^}H~1RD5G zW+tPi2nOau4F(n%3I_HMI23pS2IkHJ26ko)2F8~G28QdD)1fK=`~YJrFD(J~`QNLc zyDSAb0`DZF>k0;jL-pSqTviQq0~~~PlU0<2-Ge0p=isR}gcbn<6JV5;5Y_NnyX^M# zB3ku6dga$#Y)6NOVF?D4Wr0mJ&Kc$49i>m8`m@?G=ks^f_xj35yTR(9xppY2dwKcW za*LgO8pD`YT`_zP{uGNV50NE{6$X?mL=s;_rRyipCKy4zD8z|F-S+V*yt(IMMdkfc zdJpQ|C}o1c|GSY!p_oU+L#TQQ&pf3PNM3pJ^(jqfIl)tfIu%2L-GRvoWSqApEQ|y< zA%l}h4&X_SWTSwai^3yd!7i8eJA#}KVvXwgV|HTEX?z^Pi*qrDR zCWt(G8KO4P%&1ksCsai6Pjh>Lr3I%b2Dg*Ijij+b>2k_QfeF!BX8g2AvN@-y2%NFu zUdqU**j{^d)e=HO8ky~HbVzfm8~#9=`SUTenO$&ySo0E}rPZh^D!77>H-S;TA6QwR zvAxRGKEvkr)Mi>3pj#>k3+pWd={|LwffW^ZI99!F)X_Tg|R{Dp?fwpjT>4EzdkRWkQuG ztBrE(bQ9KC6BKLUSE#&+8@{_yA%UnhLi}p7Nj%Y|N^WsBeeSPP+0xp0<=v|7Tp?

DIgBh4>r&tis={_)vFwc zJ@CCgGq^@{W8&%TylTW~hjZuO(l+Vuc+mkyQ6vq5zGM?2$Ke9on?Z%RvSaJE3o;AS zN(B=P)6rRhQ1BN(WOlW;2oeLfd=fmHF4R@8xU@Z8}5mOK! zyF9yfc!Kc5!ZBIssvJbe6F4%dh1&WRsuwt3M8Uk6oR)FRPAgEJdgM8RkD{w6$QA>6 zl?M0Cu8-@#g$`(lT!2Ai#qlE6;XBYy4P&8FASz)kGKfMTH`WL;zLJQ`+~*8NZ=0u% z$_^aueEKH}URirxGtT)H5s6bad;n%PFdZXb8bvHn7zvh1w<%0*(lwpf<_lEM>(9Ob zpJC%Cs31j1ddH!^5eNPRckleg7=SBEAwHQCoCXcR7)52h>xUOue`ZhavKX)F^z|bk z*UHae1g05pah*c)vB0kwv>(|{iqlDkX9S5A5QGTM?azed9*bj1&yw84-^CkIf4~M+)=oIi;1X%`H^RE%kDJA z81-;>h_D1S3{SX-s)Ol4BwGtGt9FP$jt7bVu-$4C(oi#}-Rue@>w!`r>~}rDDh}PW zQZ(R7`RcndTpqjy-8K&+|An)11koc8$IoqJ@6De68b@h}V@JN1uE3sFiRGk~kDM1w z2=YzP`!oq6s)1KaA!Ir2T!GZ8xOumBGyQ_czTq-;D2_UlMi%y}5F*kM-0uCoZ!(tD z4f~hNBw+#H;r9^fjigW zFdD5$oEy9XReN!Ig;y$$i4_p zwP{o${DpMc01?Ct;tkMu*~^mTDdqAR`Ae0CU$TO~Tc7EXXL#__r;At+`O{Qhq~X13 z=t&*t*g^O5*k8gKp=9YRo&u-s%+?LO3tn#kY#!-^*)c?N<-!U@-=7%d-*w~0?*h3% z+~R0jGvxeaEmd(U)11S(dj5?Dp*RmO32jvq#`B$1PG5%Yx5mij!au(d-pgQJOb$!M z87I7%lL+JDPQAa81$dF*{9l&4Opwz+_)UsAD%Dz$UnNgN6QIoZNTHefqE89F`mGs1 z+T}I`FEE~Yz*(l+E+{O{aJDVO6k4n8!EsEAz{S1Vt15ksaoLl@iXg|+uQKEQw{=BD z($^(R&`1N<&)Rk!ArmXjXfG)ZU8D^5s4JP+>=+KF3h^d%!;>)TTF-EplqIaYuT8@K zJX8XMDKA-jmV`FGV#Fb7wge>#f{7Dat?_`L6u~68^~&k2riNvfv~D9D*`J}MUtoRM za{N_D55Q{W)Sz0BFp{GGOF6a;cHz@Q{5WTA#CfZ|*9oK$hbR{BjEY6lCw3_P0Qa}Z zq*z!YlF<}hBsBt)BN0_T zIf7b7tT_wpO}T(?-qV^ki6a=%Fc{ay&Ho)fWTcW4AqTZZ)Q(Rh4tpZIO7$$Pvl=4+ zs(5BLG(6QA3N2P<${@k{&L&kwgRO9(*W&s$$EeKsNg}R2h2w&&ZMJx$AJuG>rlQU| zT8hH~J=U_s&E;pq$Tno~st6kK;oLj6loKS+JVYyGn)33?9YKiuy_*znr06Ocv{K*1 zEkW}SzBCOk3EEO{Vi36Xoo-GW!M>L`RIR$1hUoS!CV9a^QP($DY8$8`r&>jHNy42D zDr4)L4TuotTGo|-sp)R6nTM)}p;$=0!0(UiTml+j*2N7C#k#Xl-_XA>TMmlKX=)2O zCJ)=vBl&xm^cAZ@6p?btkN{(t5d9 zLGa{nUYzRG%sQp~1oeDd2Hrx1SpVslN+0JepcK~tfn1%iS!K)eTcXY>daBJdNciV0 z5n9Hr*Q+O#+)u+U;^XHZw2J;QvtfLa4%O4hj@-t>JaI4Zuf?-8w_&fX6reiOj;VAl zLV(TG`eEHguUdxD=fvO`31!t{v-Dd!0RWqYr5c0$s(F~j4Rr%`gI&Ne?`c*?hS^!t zOXuYiKb5Y%?$xDNHoE!wtA*FNzJh5~*x|A9*e=7@9%r|_nA6&M*CUqlQPq!p>={FR zi02XH92>frv~}iHSRMp1l_Uyedc+Sh_)}JxXVPA_ zhS9iNmO2cN;p7T}p2yU3JHXtu=No$)zhoR*-l3wK~Ybf9MHc;<0JX#_L8mzR6|r zH&#x(BwOc;_3a3+EBhiXD`Pnx<8%KPN~)XHT(TL_YA>@7Mp7%4d&IEN3-czu!ogvyb(G3vZ}m_`fb(er*e-(QH~nngbd zU8bnj4kL0p;3&G^%+8YOCq%S43u{(Gwk=uT?F)jV7{%VmRY#N^^Z9C?ZlEBG$CyYd zazqf)acp?3U8Xi&S97qMiMnmT3EFmjAO>wi)Pu9Y-VK1u0=xnP-VQZLS`f_23oBpw zkQjzd(9ECE2t~yp2gZ{g5U_}2l1aM;~^rnJ5TZhVV1+UWYuHT8cIGvfo zLsLHyVCk*+@HGqO4_d9CJk7VL1FjKel?*g_btp{oA1v&C1(pm4)1e$GwU}ik9|eX9 z+Z9H|KZ&J<0%Oqe4^FLrBFe)~u-uh9d;fwLvky0#ybi6uptt0<9h;9TZ{gZg2Cebk zpdwL*J}(iZTj+2q!aMr?hpkZ$FB)C)(-}4|trTZ~zVgvXD>5ELIZ@H=noG!ZEk_1_ zeE`3>T$>a9aL?7#?ZBr0)Ml*7IMW%H@jSGA8Sisk8RByoHpwvdIh|C;0Cfg$-*ApB z7s4h%tNz~efL64$oLt+wrOSa!=y@D9Q>U>)Qhd#$Vrqj}F4a7d0=_+3tj@+a&P4+L z?=+TjHJb_Q?2COTKd*4V|5`PlUS;{HgZf5MBAGO@<4SM08T+mg{NQuUw+hm&*OXKF zh!un!nn#L&wrZm2vM_4*GJ5eh3(|6m-;nfTk83^#;iaUiH3WPNcH8S zsw$8Ff`iV??mG-kE?1vfW6ng5vDx(tobcBE@xOPadQ9+1{}=$jEft&o`o5>(4@JXs zT=uD-r*>eANt=gsloUN(z4pQuuQE_qYtj@;)7e-WOcgg|r$s9CxCTua!5PP;MVAq7ju{V8(v4N{J!6m8@mYk_pKH?_fkN+BFTr#_9h)JCE z;^^qOzf%G?&E!^ZQHro)LqWe_@|`qeh2RKpn`S8}~Rj1YAfE_sBA3^(ybu z&n_D%9ag90n;mR^5q)=<1MN{0^V+(do5`Wrx{3Y+<9(C&}u zOk2_QpJJiuX>Gs+oc)DtF%KQ)pjeTC>L(6+ziJ-WaVTB2I zRg8+tk}5g7ph=|8G;8XYyc_k8Z|~Gt!O>QB{3NF8hKcllH42n*n;cQ-$>XTg@CCjW zE#$CDzGoY!L2DD3bp_B4PN|b$HGjGOxpcTPwlWNitwy|NLeYm(PHvf)fvv{5EQTDU zpJ>eGh#v)K=!dcm4 zgh)Pn>8pW-m`@dy(DJq?JG6Z8NMVEh*J@539?6lkaKu7JbLV2A@!=5gcrJ;kAz!UB zDcmyw*7Y&@W|A*bgvI-b?ejKwW2Y>Rq;O8%tXqtUG^)iU>c#iZ3^dHkJ{baQ*4m3C z`N+!C?zd0=!oE|IZR<9r5)u<2KQnI=j*$fTL1qq1$JJv0{$Q2Orhm6n*@ z>fN%6vf~PDS=_7o0=#M>Ef&$aNRjKW{W4S}N;|@WY~!3iu8O#xa#8OM;7jc(ObXri4^Rfu z%^nh+^3?zD?wXA}{xr<>l5P3;fTz!^SOUGgOhSF{XViG)1YZUYk$t2YrmEIN9jkL{ z&`8BDi-hlYaS#)HD5C?%5MWkevjpz%nu4jN*pIV=7HI5g9ro!VVBmoy_glq=)8iaDMSE*4&^bD}_XU;^6=ubQVleLY z_Gtk_<=75c4>)@dfFS0@4oudxpoLh}T9hOx@f{^)wc=LIZ@sn2hecWI z{RH~Tx^{wFQib!T22r9>($W;@aC*Vityz0bFx9DrnTD6K09}vUSXgN&*Og&Uw_U3z z9sfmv26$*m32JtrUVGB*R;@bw^zf83^c-G{Hv5oU3%3Eln7D{w0Ed2#ZSP{s=e_>k zK7j4?vlbI*i}B1z1nvjBy!+$0T*eon<3Hi&=;uRY)wK!*2V_*tkw~P}*Cze_xGlA5 z(H}`m8oXX5OY(|8C>b{eF;W&5rM>@)UE6GAc+LAO+V%2Up_F4Ylta}RUkKxD7_2n@nTKmw#( zUsdz&zF1Ea%jV@%qgFPIUBvMS|3@Zyrggc9!4bQS~nb@J;XQ(aLJ1 zqVyX^;*Q5AwfVx&CYZwO$7Q^U1SaWh?*8qmm1e$|9I%pC7v$0yk`8gFhg?-eBpq9#$tzBI z9w;}z;FXN4Hg{mFR(p?C8P%ka_&`yszkV$zJKdOOkOu7S>x9Umo}Lllh_RTYD(MDI z)rZTx=AUPsv{JAZEpLB(g(R2l*e1>cU&N1W^9`I1c{9F*`GiNdi0FctAc=28WiBS5 zGky?qKIxyN4R|*~1Spk-F_VI|IMg&@n3>`JCZdNUn&zmDk1JQ@l>XRpxb2!yg*4El z?|c5(W2qNKa^A=L+3P2J2vWIzqiPNdQtIOe1!ZxMrZQUhGrw7>`Fs)>ch=tKoOtvG z0U|w-pJjWxcgp1oQs3%1apo+9WMcmHz3Y)r>U7ORgSCwhCxtlt*U`P35}DVI2EpV! zrtnp0J8|c|5g3*fA_N2!Y6Z-xt;mvx(jOQ=TI%<43$euPwD0WEw0?2=oq(Ncq*kKM zk9)W>b9|A%r$0%Haom`~q3;&vM@iSYMxwF#L0mM6CGL%ZacR&O(Tg4)PpJOnnFDt9 z4sKh5s?6h~A*#G4Q!3deXhyC@Xr3iBlSctuW`-vfBesV=G5P-HVLVb|Rnh)z#oH&W z?4(E&H}KA&W*8guHOiQpu`n|PcN{IgIf@9hag~H2{xcx}o?m8Zg#KfsOba8eNjJ4U zVgmW!vt2*a z0>t!`kBS*zF(^479Mfw0R{K|f#eKZbD$QG$XHm$yKG}HiKZtg`{OejpE(TgsMXvb5 zped?F`(@RY>#Y}ma5dIMr4>r$Tfv@)sI@rSKIE+?}^cYu_k) z5EHKx`NT`MDoOU?-VhaurP4_cjiw_DBaf@#c!WE+M?U=Q9GzB*-5ZN*aBjMdAdq4V z`9?ro2oaw(-Wo8O;eEl3arr{dN=S}rA84d|R#-aM_k=0B#gDaPq%Undmlinm8UYCq zTAhDT_OY!%Dsq7}P)00U-aUZ}^DBk6EfP5jd)#fCMY|apTmbIpP8pAV+D5)zO!;tqo(-&9i!MMn)(O6-)$TPQ%}@TMCr|HNi}*I$RVvn4O4Y0h4Z*Fj+-?ac4?fKn8y0l zr8`KjdeQdMl!a{k)wSt@Adb8JGP34nQIUSQDERjzdJyD09k$}`a?$7_Pro0@F8BL2ay(!Pay zu8)LANrwF0y>9VnG)8btLv?ktTJGVOW8&&S8|}4Kd;Psx9fQMzAu6LI?G@it1nsM# z6%U5dQc7@w^^R$#Q$$vEK*E^#p1)eH#zu&A8)S-R*xQM0OnP2TcUkLeQ(b z1G3SxCxHyCfk!YP*e#c2Swja2cvHy@(VSm7NC2lsrYAbxEi>PCCc~Je`|BNXNyu8BY zF7wccI=;a@CU`_qc3>0lnh37$Tm$Xvs15fPW5uW@=hU$6FNAhKnsH!;mbCesQ5&>j z9}aYott&%WKhbM>P)B^L2w$6KB+wDkW zzu#cUu{nMp0QaEkhz4;R=#K(X;UCW#>g|yP=sp!D{IVxmHkc2j`SeGYjg_8wUrpUy zUxUvEI`7TvHX~b4IOq-gL)AiPjw>a`qKfR^oIC}nbZCPTl%Zc|-3?4E74Mf`ki`Nol5tPIpUHMg{1UaU`Rf$Gc}c zT(cP8fOU_J!i`%Ws#;VD9OO zG-8`?4~TF-pDGd>xqnJvh>HD5URaE1ccLFJvkH0Z#D<>8J2>G2u}3*6xIFoU=3hRl zYLoWiHmt$EmVfIGZdWJr^N<7wECEfx&_+-w1&-V41z3<9qw|@q8W*i#q@M@9*EDMfsU*YeA!5x$=G4~39vxmw7|ZI#98_K zwIE@o|+>6@GZ6qMkw636m6XS~qhK zrNOf)F}F7Q7e+KAZlpdXDg`_m#fqo*cf%kk%2zj*&W*)1f62;MdNfIyKA`dS+pvj8 z_5G~v6A=d87VnEK~RLJS#ZBr z34x={yQ*<~yss{L)$HEotsjCTygEPJ7h|%-=mcmrmHO-c6r-gbVA|L2Zuc@DmI_Hz zXWk@Bj1MZDHJx&nddZE;MQ;b{q__@JzC?d5t%F~%J(ienZVn7QqYqAz0eISz3-v^O zet9q`@7DEX`iYutju{QP$xrX#-JR%8xTY7>18Xxjdlfx^JRh7(6SA-!55xgCGxH-` z3oYlkAt#ebzDB3lXX2XiuC-}|Y?~MR_~S5Mr!q8nn-{+n&zJS06R~pEY5GdTuDZcL zU38m^Z1Vk^X*`6e|6ZQm5`V=3*?($jUmHaT5PLfO+XK&YAKiRuq56tvZJkQc?TZ-K z?1k*wGa-o2Y&O@ez-^iBNiXw*f&_b&gFY36GFcg|#gM_@$MU#cuFJeg)Zh{mDbROdd-?9@XupZiCqpZuGi}DZ>~zIg;d<-noC4jjl?7;~I4c|< z9gfAmQaFVzTteeu--$Z#sBNg( z{Q04g3_U~@OH$Hpx)(I~=1HiuJ?SPoVZSdJMx!YoSU(omjMXUh_)9676JmgPOAx1f z;3gy8(nw}PLS+&UKsU@HoV+ZQpp?2(v660m;%o|n^b0=hgN0@|`d^x*S4}Ry<9>bh zLohubZs@>(`FRaaPo;?JL5wpR?9F-s%n1CCOxEAN&_7e7BeTyS0GRn%KdG>qIyR=x zKC+HEH}PXB5>~z+P$0OdHx?BA8?~bfZB-7I=$JWaJ!~)W^(Vn&)+e3yLM&^V1d-+G zJ~?r0cD|9-a|ns} zNe(Do0F!xn$CfU-pFynf8@6$cx5j~&G!PqY+p`b+Q3nP;X2W1OHI;xYiLHJie4Wn8 zts!Dzg@5evl~qWP(Dd^Kt-7zua@>e~@NcNYwR4gnu;xMX3M>Gbfn5F=3k3xR19Wls zN5bsltATFs+^Ld(%koHJj7b)zZpRq1$Du>)wVkcWG(Pq#06=PUV?xfW(9QEXfC38k zfQS1HC1b%5;eNh*oPUvbA2;=H(4-Z%nQ74cu14rDUinlU9KWSb_(*P zAX;km%aL?E0nX`3cElgf+xi#OQI*bI)Tsm#`^lSSA*){$*F@f$w@Ev`-!NM~JcoUh zZd^h5L-?`IYu`EHjhZii{7#pnrL*`MJ@a3>^OT2pYhoBknLa>gR*>>EmfuZmv5JJt zyogg*K*9MmGhZ4rOeM`6@v>8w|9&hUYPKmYPuNl5kriWQD*$M0B`X5m&c!q=}s*S?cpsDZ`3nG zFPe(=&0chs!gjRt`)k;rntGqSi1rj>AoJhLg%qmc9$xFr0Sh%vCZ-9=X(zE*WWdeW z)#?pZ-7H#83nlz(2B?S$f>xSIn$fvQD%I3!t6mo0z!RoDO3EF(`x**%jZ*(&T=%^*}(wGOn`HwXij|G?%O0)*E)vP>lhi}j`mrs^P?8U zsi;~&#aSg_wmtHtm3Q5lSsdmO-d-NwYAk%R+|pHu=eZF#R{nYr_`v0d^$fd%uOas~ zDyB?53dRw$`ln!!Jh6yd63Cfo1(MrdQ8~|D# zWJ;1~O}dno+yUaGE!;ie=2zy>8|Z3>ksla9qx*1?DevpWr6ulTxbdLr-%~WwCAzti z3C%IR6v#lh8^sxde<>u1HRHuuq}|Y62wjP%^b=SeLY{`0KscE%GH|<6{}t?N1X^5n zap}qL$U}1ZMVH6M zNdH3INxl~ID^0;wHDkEA%G$u?%qA2_VtLd^OF7F0Jb1}ClUGpIrqz$b9^;$f8V=(q zmcDm4!$0#I^Rj%I*Up4r>1#W85B`RVEaKM>Yz6xF zryQ}~gIO-QE`G@r?UyjKDH{E+U|MO}Oe)6VyM z$qOg`U)z>#8qPHW75^(KF)vSClhrqR#T08dDPd+%*e(@~dft`G_1=9=c9?EJDru+m zO0a7ECt`8OI3C+b+}WmnI=Ez5KQ416fI{Y%Wjoz9`TXMfT+|TNv7=Pf5dYS5a6B*Z z2qA@mZLGRhvHIDiuTW$SJWwgr8&{^>QgLNL;!OIpmvly=*%-Y>(W6fp#F)zAeXUGJ zrmF1R2gHfroz*|`zn|^DP0GAx@ohtv8P?f_$7~zuPsB7s$1+|8c*ufE)hvH^c88&! zrP44(09~y%us775y(whuZ=cp8YGk6x{ABWhBQXD76-3Y_9%K>$%mkXNAj>iHqjri> zJI{oU--@_^Le1W?Qc%yRWIJQStt8rA4T&Bj-};lN0(BozHhOb$OJMB*mYxRcG zr5lY=+3;l=Pf+CI6}sFB>1JMez0Hk3oXUc$)BaT)X^^|ViuBUY^(@fz zA1cN&OHhE=q|>j-habYU3>5+bh1NFuKxeU1ZP94Q!&*Rlzr-Ue#S`Uu#anEWN8w16 zihDNh)!D!9S6H`INLNEuAU3;i&6_+$X76(tO^mMUU$uv0f@k;R=)S8~MCevZLKkJyKhrnk%+ia~Cq5G}N?;8w>hZe}R7qb%~p*by5?Y z4Jw7dY8s9jG#~dY*7$}S!4%f+EF8Xx7E90ikHe0$@C!Pwc6$cLn$$!nKcn-aLHt7d zrR;0P`7raHb(P`H;j%VvqeDikUxL6f*CTZo~{WCXDY`tv)a|5*_THK z`nF9}YNMUrohNndINI&clg(6JI{LlEY=nBkKB_#_yk6vk!uIwi`{bnD*CUP3-%#){ z7PrrsMV|>hJ`V#?Z$zMEWs=Co?VmVwH;8}J1QR(%-gFaJH-%L>WGlkKNoi1gO_?^= zC={(PyPmY;hDwoM*I`p!G~95Mx3gR={09df6nW5JO$)d(a>zs@Xx6cJ`f?^iXbb42 zF?#TiSX~@1;b+}* zo|DrZBi{h~4Rd!eg^Zde_uMPxv3OP+d}pl)h9fvpAC6%NKeu9emut_M^PA*!@kAk> zhR2gq&ak_ZlvJ}o{p9Ku!_Xrx6O$N7GzLwU+pgKwWisAcna!QM$op5%(V^rJ9yL#C zLT$A9diuvE;47~+X>CmKg5iq*s6RMtT~4d(T8pEPq08EI|hQunMddm4pP zDdVGrd81eRU>B1h@c3MWBt9r~80)BQ4LlouX70^V*iOnb)?=~yIra5DJUuONOxWJm zWZooTjc!Gj(tXOjEZTDVzG+(N`n4;09~qsC`04iclx#QS!lRK99+=W|rvRJnS(t=1 zj>JbBqQ}btl>kpE<1pO@U*T4kZxJ`9sDLe|Y4?Vc|>Rdt?hYZhQ@n4egA@2xZpi4e4YE8nU z9c7dZo*K_Jc7hreIl=3B4D4fXPut0=MPpQ<`G&533re75v$dp8yQ!bdS)=GQL8-~R zrg%)h=&Pa_sDx=1A-uLK{|m7Yeb1%+ulQ|k;c_KM-yt&bkWyYj^wWgfb$otiCOH!c z5k&(NuB6}O>&-mvoW~a{XCiLRRxIWqwK*|QhD*vgcxZ+jA?y$RnanCF)ib$l;1SoZ zo->hGB}KmREe~i@Q(PpH+t;-F15AF|77QNw+tb`pc}uFPToz890-z zp+CTQpu~hCk1N?pvdG{$eiKf@DoA|QkL!%ImF|9>AN5T)DpxeoWR?7IGmP1Fg_~s8rrQn5{2&Q zL#9QThO$%fV!$|Q!wQfgtiP_eC0bJ~s~=sKL*$kndwW;31V&j1;<8EZSI$1_zpk$# zHo8;@FOuydL^5Bl|Lx2kSoi0mg(c!xwp%>=gmA*9l`bhTFIm^NhJj$UP6nNl#0^qu zUk#UcVwJ8ss%e!Yz5F(Ah5s_LT9I2rz}N(%$T=fbEHxRT+Wdn06@fgkL9Dx509T{v zy!G$vf0(Ic7>5@1tE)mfmWXwSUkvY)$NTPn=`i8bPX&dq{IApIayEErQS`5dxy$U( zWRS@kr9(()@vWe1a0n-&hM{MG1o&aRkSAq-guBpF4!p~@+klWeR9_5)$`3|%s$+M$GK9imm7#Wp$Q ztB&g6-WMyqORr$`m=x{QO1BG_43QN%)lLn4wWu0+yKpjs^h+~r5)%nPc3^pK<^MFj zDj@NiSu67b80HSH8{36N->^ z#`>ohGTCg3msfKLGnBGIR2&ZoLZ|jOQ>UC^!Nwk2y#w-TXQ?3P&EDR*`RSvsIn#K>A zEqfl;fP}ZB;eD*>I&zIo^Dg5Gy4^O^Ml1V}qz_ zeGVfyJaxPMy-7IjE6LY~@HnSIkvH_L`tQf#72-Cj$PsI1;Dng@IYE{@Gsw7+yR%5m zp=<8{^g??!Rz~wb-k2ho!{8**%7$WNf~Y)~#bu=08<(&SS>KUukZo5%9Tc`!C6#0)_n{<49JJgHKo$|jm@-gpAkZR_=5 z;Yuwi!$e<*VEk{$Hql9825b=Ryo<0jCP%jmG-`4}ELYfnI=D;CB?g3|fqlRX0z4DW z{mF@GD5H=_T4>k8_dWt;R2a5*iT(0+=T5e+_z0;jYJ9f~smXzflRxZ5%C3#c z6x^!lPiYD@7A%R{tuV;dy}~A0+@5wBX|rMG1LsV<2M2;0HHu0_;+x|s2jC_U*NBDo z5|Od5Gwe!hJ<@kXENz=TY*31=A)r}C zs{J)FrX8Xi#llnF`AiCWkPJTqfl8&;@Y_~MoQ7<&G-)K*_Fw~(zShzWr7v8=OB@?o zMsH_ti$bjxOA?K&-U`ebzjao-*}#KBZ3(gN;~-Z+HBxHfhjRJcIC_YH?7u-`wCzmp z+Pk%Q?#2kBL<^|X(8=D2SQcEBJdmIm1ZmLo&BN)1tEuH?iku9(@QtDikMb|>1KBxb z+@dlm{3fp`ovFL@6H~z#R1~^mXf70xBxh}12J$WV{P^wJt@~f5^o>EH~ zAaHm)SwATdiHU3j_F?RpjkJQ6@Q+Zec6h%S^rNTCSm{KNr^@uha6C7AcJ(@G!$*|e z$C|Z@W47n$UF+n)4mw4Y1awHNd85q_i5h`8xSRCyAUaB%1UT6<(Kl$`WTjMjo??e8X`DIk=+>#;&yLnVQ*&@607|Jdld1nXGiWILD_O0tgq$%$p3;*cC~2>4_J z*bgk8kpbB%zH4`haLOra!6 zf|ZPyOJ=!Y8f6ZCQH_r+y9m$ueqZys-#PZl0O?Qo%xUaVy3Cn6N-paH13cBU)>@49 zx0gaY_O51BH&_Hg77AcVY%d5MB@~nsSy)|#WIQLZMW`F%odUG1)H&0U?A6SQY!sI) zGFdl@Xcz@a*<|cq2xC5$@Ek1~^+IaHu+f_geMD<0Ca{qpK{6%G^hdl(3G(!)q`^os z8I0UWh#jzvz!Y>UMW{J8)*>PEqY@b+xDdEtq!5D2HcEM0eiA6kXuvr6cw`92en!0r z7a9nfQq=SyNt(p_|KHqvLSkGTMNr=qWdqL=Q@Kg%xS5-{S@4;;SO7aPHWpSMMiy>H zHXaQYRz6NPK29Eb78X7hmUlF~$NzPLy`%XzOYi^tgjmG_S>Ocx|E{3!XzAu@;$i_N kYVK%iK_+W&Vr8LbVPfv>JZACVlh0tXl1dUaV#dM$1FH@ft^fc4 literal 0 HcmV?d00001 diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts index bd064f4b249f..08087a72676f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts @@ -23,6 +23,7 @@ import amundsen from '../assets/img/service-icon-amundsen.png'; import athena from '../assets/img/service-icon-athena.png'; import atlas from '../assets/img/service-icon-atlas.svg'; import azuresql from '../assets/img/service-icon-azuresql.png'; +import bigtable from '../assets/img/service-icon-bigtable.png'; import clickhouse from '../assets/img/service-icon-clickhouse.png'; import couchbase from '../assets/img/service-icon-couchbase.svg'; import dagster from '../assets/img/service-icon-dagster.png'; @@ -122,6 +123,7 @@ export const SQLITE = sqlite; export const MSSQL = mssql; export const REDSHIFT = redshift; export const BIGQUERY = query; +export const BIGTABLE = bigtable; export const HIVE = hive; export const IMPALA = impala; export const POSTGRES = postgres; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts index 31c016818187..bf348fcd5916 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DatabaseServiceUtils.ts @@ -17,6 +17,7 @@ import { DatabaseServiceType } from '../generated/entity/services/databaseServic import athenaConnection from '../jsons/connectionSchemas/connections/database/athenaConnection.json'; import azureSQLConnection from '../jsons/connectionSchemas/connections/database/azureSQLConnection.json'; import bigQueryConnection from '../jsons/connectionSchemas/connections/database/bigQueryConnection.json'; +import bigTableConnection from '../jsons/connectionSchemas/connections/database/bigTableConnection.json'; import clickhouseConnection from '../jsons/connectionSchemas/connections/database/clickhouseConnection.json'; import couchbaseConnection from '../jsons/connectionSchemas/connections/database/couchbaseConnection.json'; import customDatabaseConnection from '../jsons/connectionSchemas/connections/database/customDatabaseConnection.json'; @@ -71,6 +72,11 @@ export const getDatabaseConfig = (type: DatabaseServiceType) => { break; } + case DatabaseServiceType.BigTable: { + schema = bigTableConnection; + + break; + } case DatabaseServiceType.Clickhouse: { schema = clickhouseConnection; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts index b6af88398ad3..5fee6fdf94ff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtilClassBase.ts @@ -22,6 +22,7 @@ import { ATLAS, AZURESQL, BIGQUERY, + BIGTABLE, CLICKHOUSE, COMMON_UI_SCHEMA, COUCHBASE, @@ -160,6 +161,9 @@ class ServiceUtilClassBase { case DatabaseServiceType.BigQuery: return BIGQUERY; + case DatabaseServiceType.BigTable: + return BIGTABLE; + case DatabaseServiceType.Hive: return HIVE;