Skip to content

Commit

Permalink
fix(airflow): Add comma parsing of owners to DataJobs (#10903)
Browse files Browse the repository at this point in the history
  • Loading branch information
eboneil committed Jul 17, 2024
1 parent 998d7d1 commit 0b64de8
Showing 1 changed file with 11 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ def _get_dependencies(
)
return upstream_tasks

@staticmethod
def _extract_owners(dag: "DAG") -> List[str]:
return [owner.strip() for owner in dag.owner.split(",")]

@staticmethod
def generate_dataflow(
config: DatahubLineageConfig,
Expand Down Expand Up @@ -175,7 +179,7 @@ def generate_dataflow(
data_flow.url = f"{base_url}/tree?dag_id={dag.dag_id}"

if config.capture_ownership_info and dag.owner:
owners = [owner.strip() for owner in dag.owner.split(",")]
owners = AirflowGenerator._extract_owners(dag)
if config.capture_ownership_as_group:
data_flow.group_owners.update(owners)
else:
Expand Down Expand Up @@ -282,10 +286,12 @@ def generate_datajob(
datajob.url = f"{base_url}/taskinstance/list/?flt1_dag_id_equals={datajob.flow_urn.flow_id}&_flt_3_task_id={task.task_id}"

if capture_owner and dag.owner:
if config and config.capture_ownership_as_group:
datajob.group_owners.add(dag.owner)
else:
datajob.owners.add(dag.owner)
if config and config.capture_ownership_info:
owners = AirflowGenerator._extract_owners(dag)
if config.capture_ownership_as_group:
datajob.group_owners.update(owners)
else:
datajob.owners.update(owners)

if capture_tags and dag.tags:
datajob.tags.update(dag.tags)
Expand Down

0 comments on commit 0b64de8

Please sign in to comment.