chore(ingest/presto-on-hive) Set enable_properties_merge to True by d…

…efault (datahub-project#10469)

metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py

@@ -130,8 +130,8 @@ class HiveMetastore(BasicSQLAlchemyConfig):
  )
 
  enable_properties_merge: bool = Field(
- default=False,
- description="By default, the connector overwrites properties every time. Set this to True to enable merging of properties with what exists on the server.",
+ default=True,
+ description="By default, the connector enables merging of properties with what exists on the server. Set this to False to enable the default connector behavior of overwriting properties on each ingestion.",
  )
 
  simplify_nested_field_paths: bool = Field(

metadata-ingestion/src/datahub/testing/compare_metadata_json.py

@@ -5,6 +5,7 @@
 import os
 import pathlib
 import pprint
+import re
 import shutil
 import tempfile
 from typing import Any, Dict, List, Sequence, Union
@@ -40,6 +41,7 @@ def assert_metadata_files_equal(
  update_golden: bool,
  copy_output: bool,
  ignore_paths: Sequence[str] = (),
+ ignore_paths_v2: Sequence[str] = (),
  ignore_order: bool = True,
 ) -> None:
  golden_exists = os.path.isfile(golden_path)
@@ -70,6 +72,16 @@ def assert_metadata_files_equal(
  logger.info(f"Error reformatting golden file as MCP/MCEs: {e}")
  golden = load_json_file(golden_path)
 
+ if ignore_paths_v2:
+ golden_json = load_json_file(golden_path)
+ for i, obj in enumerate(golden_json):
+ aspect_json = obj.get("aspect", {}).get("json", [])
+ for j, item in enumerate(aspect_json):
+ if isinstance(item, dict):
+ if item.get("path") in ignore_paths_v2:
+ json_path = f"root[{i}]['aspect']['json'][{j}]['value']"
+ ignore_paths = (*ignore_paths, re.escape(json_path))
+
  diff = diff_metadata_json(output, golden, ignore_paths, ignore_order=ignore_order)
  if diff and update_golden:
  if isinstance(diff, MCPDiff):