Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(ingest/dbt): always encode tag urns #10799

Merged
merged 3 commits into from
Jul 11, 2024
Merged

fix(ingest/dbt): always encode tag urns #10799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f9aa820
fix(ingest/dbt): always encode tag urns
hsheth2 Jun 27, 2024
6fa24d6
tweak
hsheth2 Jul 1, 2024
5f484c5
Merge branch 'master' into db-tags
hsheth2 Jul 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion metadata-ingestion/src/datahub/emitter/mce_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ def get_or_add_aspect(mce: MetadataChangeEventClass, default: Aspect) -> Aspect:

def make_global_tag_aspect_with_tag_list(tags: List[str]) -> GlobalTagsClass:
return GlobalTagsClass(
tags=[TagAssociationClass(f"urn:li:tag:{tag}") for tag in tags]
tags=[TagAssociationClass(make_tag_urn(tag)) for tag in tags]
)


Expand Down
13 changes: 8 additions & 5 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1483,7 +1483,7 @@ def get_patched_mce(self, mce):
transformed_tag_list = self.get_transformed_tags_by_prefix(
tag_aspect.tags,
mce.proposedSnapshot.urn,
mce_builder.make_tag_urn(self.config.tag_prefix),
tag_prefix=self.config.tag_prefix,
)
tag_aspect.tags = transformed_tag_list

Expand Down Expand Up @@ -1874,16 +1874,19 @@ def get_transformed_tags_by_prefix(
self,
new_tags: List[TagAssociationClass],
entity_urn: str,
tags_prefix_filter: str,
tag_prefix: str,
) -> List[TagAssociationClass]:
tag_set = {new_tag.tag for new_tag in new_tags}

if self.ctx.graph:
existing_tags_class = self.ctx.graph.get_tags(entity_urn)
if existing_tags_class and existing_tags_class.tags:
for exiting_tag in existing_tags_class.tags:
if not exiting_tag.tag.startswith(tags_prefix_filter):
tag_set.add(exiting_tag.tag)
for existing_tag in existing_tags_class.tags:
if tag_prefix and existing_tag.tag.startswith(
mce_builder.make_tag_urn(tag_prefix)
):
continue
tag_set.add(existing_tag.tag)
return [TagAssociationClass(tag) for tag in sorted(tag_set)]

# This method attempts to read-modify and return the glossary terms of a dataset.
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/unit/test_dbt_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ def test_dbt_source_patching_tags():
["new_non_dbt", "dbt:new_dbt"]
)
transformed_tags = source.get_transformed_tags_by_prefix(
new_tag_aspect.tags, "urn:li:dataset:dummy", "urn:li:tag:dbt:"
new_tag_aspect.tags, "urn:li:dataset:dummy", "dbt:"
)
expected_tags = {
"urn:li:tag:new_non_dbt",
Expand Down
Loading