Skip to content

Commit

Permalink
fix(ingestion/lite): An index with the name aspect_idxalready exists … (
Browse files Browse the repository at this point in the history
  • Loading branch information
jonasHanhan authored and sleeperdeep committed Jun 25, 2024
1 parent e8c0a7d commit 8faf5b1
Showing 1 changed file with 19 additions and 5 deletions.
24 changes: 19 additions & 5 deletions metadata-ingestion/src/datahub/lite/duckdb_lite.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,34 @@ def __init__(self, config: DuckDBLiteConfig) -> None:
if not config.read_only:
self._init_db()

def _init_db(self):
def _create_unique_index(
self, index_name: str, table_name: str, columns: list
) -> None:
try:
self.duckdb_client.execute(
f"CREATE UNIQUE INDEX {index_name} ON {table_name} ({', '.join(columns)})"
)
except duckdb.CatalogException as e:
if "already exists" not in str(e).lower():
raise

def _init_db(self) -> None:
self.duckdb_client.execute(
"CREATE TABLE IF NOT EXISTS metadata_aspect_v2 "
"(urn VARCHAR, aspect_name VARCHAR, version BIGINT, metadata JSON, system_metadata JSON, createdon BIGINT)"
)
self.duckdb_client.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS aspect_idx ON metadata_aspect_v2 (urn, aspect_name, version)"

self._create_unique_index(
"aspect_idx", "metadata_aspect_v2", ["urn", "aspect_name", "version"]
)

self.duckdb_client.execute(
"CREATE TABLE IF NOT EXISTS metadata_edge_v2 "
"(src_id VARCHAR, relnship VARCHAR, dst_id VARCHAR, dst_label VARCHAR)"
)
self.duckdb_client.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS edge_idx ON metadata_edge_v2 (src_id, relnship, dst_id)"

self._create_unique_index(
"edge_idx", "metadata_edge_v2", ["src_id", "relnship", "dst_id"]
)

def location(self) -> str:
Expand Down

0 comments on commit 8faf5b1

Please sign in to comment.