Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(ingest): ensure workunits are created for all LookML views #2965

Merged
merged 7 commits into from
Jul 29, 2021
Merged

fix(ingest): ensure workunits are created for all LookML views #2965

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ef27728
fix(ingest): ensure workunits are created for all LookML views
jameslamb Jul 28, 2021
68e8fe5
move caching inside get_workunits()
jameslamb Jul 28, 2021
1b0d76a
revert unnecessary changes
jameslamb Jul 28, 2021
91f1bcf
revert more unnecessary diff
jameslamb Jul 28, 2021
89fcfbf
remove even more unnecessary diff
jameslamb Jul 28, 2021
b024afd
Merge branch 'master' into fix/lookml-extends
jameslamb Jul 28, 2021
cabaa9f
Update metadata-ingestion/src/datahub/ingestion/source/lookml.py
jameslamb Jul 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions metadata-ingestion/src/datahub/ingestion/source/lookml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from dataclasses import field as dataclass_field
from dataclasses import replace
from enum import Enum
from typing import Any, Dict, Iterable, List, Optional, Tuple
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple

import pydantic

Expand Down Expand Up @@ -209,6 +209,7 @@ def _load_viewfile(
looker_viewfile = LookerViewFile.from_looker_dict(
path, parsed, self._base_folder, reporter
)
logger.debug(f"adding viewfile for path {path} to the cache")
self.viewfile_cache[path] = looker_viewfile
return looker_viewfile
except Exception as e:
Expand Down Expand Up @@ -607,6 +608,10 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
str(self.source_config.base_folder), self.reporter
)

# some views can be mentioned by multiple 'include' statements, so this set is used to prevent
# creating duplicate MCE messages
views_with_workunits: Set[str] = set()

# The ** means "this directory and all subdirectories", and hence should
# include all the files we want.
model_files = sorted(self.source_config.base_folder.glob("**/*.model.lkml"))
Expand All @@ -628,8 +633,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
continue

for include in model.resolved_includes:
is_view_seen = viewfile_loader.is_view_seen(include)
if is_view_seen:
if include in views_with_workunits:
continue

logger.debug(f"Attempting to load view file: {include}")
Expand Down Expand Up @@ -663,6 +667,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]:
id=f"lookml-{maybe_looker_view.view_name}", mce=mce
)
self.reporter.report_workunit(workunit)
views_with_workunits.add(include)
yield workunit
else:
self.reporter.report_views_dropped(
Expand Down