Restructured dev and test code

datahub-classify/src/datahub_classify/infotype_predictor.py

@@ -8,15 +8,17 @@
 logger = logging.getLogger(__name__)
 
 
-def get_infotype_function_mapping(infotypes, global_config):
+def get_infotype_function_mapping(
+    infotypes: Optional[list], global_config: dict
+) -> dict:
     from inspect import getmembers, isfunction
 
     module_name = "datahub_classify.infotype_helper"
     module = importlib.import_module(module_name)
     module_fn_dict = dict(getmembers(module, isfunction))
     infotype_function_map = {}
     if not infotypes:
-        infotypes = global_config.keys()
+        infotypes = list(global_config.keys())
     for infotype in infotypes:
         if infotype not in global_config.keys():
             logger.warning(f"Configuration is not available for infotype - {infotype}")

datahub-classify/src/datahub_classify/infotype_utils.py

@@ -1,14 +1,16 @@
 import logging
 import re
+from typing import Optional
 
 from datahub_classify.constants import PREDICTION_FACTORS_AND_WEIGHTS, VALUES
+from datahub_classify.helper_classes import Metadata
 
 logger = logging.getLogger(__name__)
 
 
 # TODO: Exception handling
 # Match regex for Name and Description
-def match_regex(text_to_match, regex_list):
+def match_regex(text_to_match: str, regex_list: list) -> float:
     original_text = text_to_match.lower()
     cleaned_text = "".join(e for e in original_text if e.isalpha())
     match_score: float = 0
@@ -36,7 +38,7 @@ def match_regex(text_to_match, regex_list):
 
 
 # Match data type
-def match_datatype(dtype_to_match, dtype_list):
+def match_datatype(dtype_to_match: str, dtype_list: list[str]) -> int:
     dtype_list = [str(s).lower() for s in dtype_list]
     dtype_to_match = dtype_to_match.lower()
     if dtype_to_match in dtype_list:
@@ -47,7 +49,7 @@ def match_datatype(dtype_to_match, dtype_list):
 
 
 # Match regex for values
-def match_regex_for_values(values, regex_list):
+def match_regex_for_values(values: list, regex_list: list) -> float:
     values_score_list = []
     length_values = len(values)
     values = [str(x).lower() for x in values]
@@ -66,7 +68,9 @@ def match_regex_for_values(values, regex_list):
     return values_score
 
 
-def detect_named_entity_spacy(spacy_models_list, entities_of_interest, value):
+def detect_named_entity_spacy(
+    spacy_models_list: list, entities_of_interest: list[str], value: str
+) -> bool:
     for spacy_model in spacy_models_list:
         doc = spacy_model(value)
         for ent in doc.ents:
@@ -75,7 +79,9 @@ def detect_named_entity_spacy(spacy_models_list, entities_of_interest, value):
     return False
 
 
-def perform_basic_checks(metadata, values, config_dict, infotype=None):
+def perform_basic_checks(
+    metadata: Metadata, values: list, config_dict: dict, infotype: Optional[str] = None
+) -> bool:
     basic_checks_status = True
     minimum_values_threshold = 50
     if (