fix(ingestion): ingest emails as empty if no ldap attribute (#9433)

Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
datahub-project · Jul 1, 2024 · 93616f7 · 93616f7
1 parent 7fb56ef
commit 93616f7
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 45 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/ldap.py b/metadata-ingestion/src/datahub/ingestion/source/ldap.py
@@ -334,10 +334,12 @@ def handle_user(self, dn: str, attrs: Dict[str, Any]) -> Iterable[MetadataWorkUn
  manager_ldap = guess_person_ldap(m_attrs, self.config, self.report)
 
  m_email = get_attr_or_none(
- m_attrs, self.config.user_attrs_map["email"], manager_ldap
+ m_attrs, self.config.user_attrs_map["email"]
  )
  make_manager_urn = (
- m_email if self.config.use_email_as_username else manager_ldap
+ m_email
+ if m_email and self.config.use_email_as_username
+ else manager_ldap
  )
 
  except ldap.LDAPError as e:
@@ -377,7 +379,7 @@ def build_corp_user_mce(
  last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode()
  groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"])
 
- email = get_attr_or_none(attrs, self.config.user_attrs_map["email"], ldap_user)
+ email = get_attr_or_none(attrs, self.config.user_attrs_map["email"])
  display_name = get_attr_or_none(
  attrs, self.config.user_attrs_map["displayName"], full_name
  )
@@ -404,7 +406,9 @@ def build_corp_user_mce(
 
  manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None
 
- make_user_urn = email if self.config.use_email_as_username else ldap_user
+ make_user_urn = (
+ email if email and self.config.use_email_as_username else ldap_user
+ )
 
  user_snapshot = CorpUserSnapshotClass(
  urn=f"urn:li:corpuser:{make_user_urn}",
@@ -438,17 +442,17 @@ def build_corp_group_mce(self, attrs: dict) -> Optional[MetadataChangeEvent]:
  admins = parse_users(attrs, self.config.group_attrs_map["admins"])
  members = parse_users(attrs, self.config.group_attrs_map["members"])
 
- email = get_attr_or_none(
- attrs, self.config.group_attrs_map["email"], full_name
- )
+ email = get_attr_or_none(attrs, self.config.group_attrs_map["email"])
  description = get_attr_or_none(
  attrs, self.config.group_attrs_map["description"]
  )
  displayName = get_attr_or_none(
  attrs, self.config.group_attrs_map["displayName"]
  )
 
- make_group_urn = email if self.config.use_email_as_username else full_name
+ make_group_urn = (
+ email if email and self.config.use_email_as_username else full_name
+ )
 
  group_snapshot = CorpGroupSnapshotClass(
  urn=f"urn:li:corpGroup:{make_group_urn}",

diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json
@@ -6,7 +6,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "simpons-group",
  "admins": [],
  "members": [
  "urn:li:corpuser:hsimpson",
@@ -20,7 +19,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -35,7 +35,6 @@
  },
  "active": true,
  "displayName": "Bart Simpson",
- "email": "bsimpson",
  "title": "Mr. Boss",
  "firstName": "Bart",
  "lastName": "Simpson",
@@ -52,7 +51,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -87,7 +87,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -102,7 +103,6 @@
  },
  "active": true,
  "displayName": "Lisa Simpson",
- "email": "lsimpson",
  "firstName": "Lisa",
  "lastName": "Simpson",
  "fullName": "Lisa Simpson"
@@ -118,7 +118,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -133,7 +134,6 @@
  },
  "active": true,
  "displayName": "Maggie Simpson",
- "email": "msimpson",
  "firstName": "Maggie",
  "lastName": "Simpson",
  "fullName": "Maggie Simpson"
@@ -149,7 +149,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -164,7 +165,6 @@
  },
  "active": true,
  "displayName": "Hester Bevan",
- "email": "hbevan",
  "firstName": "Hester",
  "lastName": "Bevan",
  "fullName": "Hester Bevan"
@@ -180,7 +180,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -195,7 +196,6 @@
  },
  "active": true,
  "displayName": "Evalyn Haas",
- "email": "ehaas",
  "firstName": "Evalyn",
  "lastName": "Haas",
  "fullName": "Evalyn Haas"
@@ -211,7 +211,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -221,7 +222,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "HR Department",
  "admins": [],
  "members": [],
  "groups": []
@@ -232,7 +232,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -242,7 +243,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "Finance Department",
  "admins": [],
  "members": [],
  "groups": []
@@ -253,7 +253,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -268,7 +269,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -283,7 +285,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -298,7 +301,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -313,7 +317,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -328,7 +333,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -343,7 +349,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -358,7 +365,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -373,7 +381,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -388,7 +397,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 }
 ]
diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json
@@ -6,7 +6,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "HR Department",
  "admins": [],
  "members": [],
  "groups": []

diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json
@@ -9,7 +9,6 @@
  "customProperties": {},
  "active": true,
  "displayName": "Bart Simpson",
- "email": "bsimpson",
  "title": "Mr. Boss",
  "firstName": "Bart",
  "lastName": "Simpson",

diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json
@@ -6,7 +6,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "HR Department",
  "admins": [],
  "members": [],
  "groups": []
@@ -29,7 +28,6 @@
  "aspects": [
  {
  "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
- "email": "Finance Department",
  "admins": [],
  "members": [],
  "groups": []

diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json
@@ -9,7 +9,6 @@
  "customProperties": {},
  "active": true,
  "displayName": "Bart Simpson",
- "email": "bsimpson",
  "title": "Mr. Boss",
  "firstName": "Bart",
  "lastName": "Simpson",

diff --git a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json
@@ -9,7 +9,6 @@
  "customProperties": {},
  "active": true,
  "displayName": "Hester Bevan",
- "email": "hbevan",
  "firstName": "Hester",
  "lastName": "Bevan",
  "fullName": "Hester Bevan"
@@ -28,7 +27,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -41,7 +41,6 @@
  "customProperties": {},
  "active": true,
  "displayName": "Evalyn Haas",
- "email": "ehaas",
  "firstName": "Evalyn",
  "lastName": "Haas",
  "fullName": "Evalyn Haas"
@@ -60,7 +59,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -75,7 +75,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 },
 {
@@ -90,7 +91,8 @@
  },
  "systemMetadata": {
  "lastObserved": 1615443388097,
- "runId": "ldap-test"
+ "runId": "ldap-test",
+ "lastRunId": "no-run-id-provided"
  }
 }
 ]