Fix #9116 & #8284 - Clean tableau source, fix ownership, add descript…

…ion and SSL verification (#9241)

Fix #9116 & #8284 - Clean tableau source, fix ownership, add description and SSL verification (#9241)

.pylintrc

@@ -6,7 +6,8 @@
 # R0901: too-many-ancestors - We are already inheriting from SQA classes with a bunch of ancestors
 # W0703: broad-except - We are dealing with many different source systems, but we want to make sure workflows run until the end
 # W0511: fixme - These are internal notes and guides
-disable=W1203,W1202,R0903,W0707,R0901,W1201,W0703,W0511
+# W1518: method-cache-max-size-none - allow us to use LRU Cache with maxsize `None` to speed up certain calls
+disable=W1203,W1202,R0903,W0707,R0901,W1201,W0703,W0511,W1518
 
 docstring-min-length=20
 max-args=7

ingestion/setup.py

@@ -140,7 +140,7 @@ def get_long_description():
     "snowflake-usage": {"snowflake-sqlalchemy~=1.4.3"},
     "sample-entity": {"faker~=8.1.1"},
     "superset": {},
-    "tableau": {"tableau-api-lib==0.1.29"},
+    "tableau": {"tableau-api-lib==0.1.50"},
     "vertica": {"sqlalchemy-vertica[vertica-python]>=0.0.5"},
     "webhook-server": {},
     "salesforce": {"simple_salesforce~=1.11.4"},

ingestion/src/airflow_provider_openmetadata/hooks/openmetadata.py

@@ -21,14 +21,14 @@
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     AuthProvider,
     OpenMetadataConnection,
-    VerifySSL,
 )
 from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
     OpenMetadataJWTClientConfig,
 )
 from metadata.generated.schema.security.ssl.validateSSLClientConfig import (
     ValidateSSLClientConfig,
 )
+from metadata.generated.schema.security.ssl.verifySSLConfig import VerifySSL
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
 

ingestion/src/metadata/ingestion/ometa/mixins/user_mixin.py

@@ -13,11 +13,14 @@
 
 To be used by OpenMetadata class
 """
+import traceback
+from functools import lru_cache
 from typing import Optional
 
 from metadata.generated.schema.entity.teams.user import User
 from metadata.ingestion.ometa.client import REST
 from metadata.ingestion.ometa.utils import ometa_logger
+from metadata.utils.elasticsearch import ES_INDEX_MAP
 
 logger = ometa_logger()
 
@@ -31,17 +34,42 @@ class OMetaUserMixin:
 
     client: REST
 
-    def get_user_by_email(self, email: str) -> Optional[User]:
+    email_search = (
+        "/search/query?q=email:{email}&from={from_}&size={size}&index="
+        + ES_INDEX_MAP[User.__name__]
+    )
+
+    @lru_cache(maxsize=None)
+    def get_user_by_email(
+        self,
+        email: Optional[str],
+        from_count: int = 0,
+        size: int = 10,
+    ) -> Optional[User]:
         """
         GET user entity by name
 
-        :param email: User Email
+        Args:
+            email: user email to search
+            from_count: records to expect
+            size: number of records
         """
         if email:
 
-            name = email.split("@")[0]
-            users = self.es_search_from_fqn(entity_type=User, fqn_search_string=name)
-            for user in users or []:
-                if user.email.__root__ == email:
+            query_string = self.email_search.format(
+                email=email, from_=from_count, size=size
+            )
+
+            try:
+                entity_list = self._search_es_entity(
+                    entity_type=User, query_string=query_string
+                )
+                for user in entity_list or []:
                     return user
+            except Exception as err:
+                logger.debug(traceback.format_exc())
+                logger.warning(
+                    f"Could not get user info from ES for user email {email} due to {err}"
+                )
+
         return None

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -95,12 +95,9 @@
     InvalidAuthProviderException,
     auth_provider_registry,
 )
-from metadata.ingestion.ometa.ssl_registry import (
-    InvalidSSLVerificationException,
-    ssl_verification_registry,
-)
 from metadata.ingestion.ometa.utils import get_entity_type, model_str, ometa_logger
 from metadata.utils.secrets.secrets_manager_factory import SecretsManagerFactory
+from metadata.utils.ssl_registry import get_verify_ssl_fn
 
 logger = ometa_logger()
 
@@ -190,20 +187,14 @@ def __init__(self, config: OpenMetadataConnection, raw_data: bool = False):
 
         self._auth_provider = auth_provider_fn(self.config)
 
-        get_verify_ssl = ssl_verification_registry.registry.get(
-            self.config.verifySSL.value
-        )
-        if not get_verify_ssl:
-            raise InvalidSSLVerificationException(
-                f"Cannot find {self.config.verifySSL.value} in {ssl_verification_registry.registry}"
-            )
+        get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
 
         client_config: ClientConfig = ClientConfig(
             base_url=self.config.hostPort,
             api_version=self.config.apiVersion,
             auth_header="Authorization",
             auth_token=self._auth_provider.get_access_token,
-            verify=get_verify_ssl(config),
+            verify=get_verify_ssl(self.config.sslConfig),
         )
         self.client = REST(client_config)
         self._use_raw_data = raw_data