Merge pull request #424 from hellohaptik/develop

Develop to Master Sync

datastore/elastic_search/query.py

@@ -8,13 +8,14 @@
 import warnings
 
 from six import string_types
+from six.moves import range
+from six.moves import zip
 
 from datastore import constants
+from datastore.exceptions import DataStoreRequestException
 from external_api.constants import SENTENCE, ENTITIES
 from language_utilities.constant import ENGLISH_LANG
 from lib.nlp.const import TOKENIZER
-from six.moves import range
-from six.moves import zip
 
 # Local imports
 
@@ -213,9 +214,14 @@ def get_entity_unique_values(connection, index_name, doc_type, entity_name, valu
 
     if value_search_term:
         data['query']['bool']['minimum_should_match'] = 1
+        _search_field = 'value'
+        _search_term = value_search_term.lower()
+        if len(_search_term.split()) > 1:
+            # terms with spaces in them do not support wild queries on analyzed fields
+            _search_field = 'value.keyword'
         data['query']['bool']['should'].append({
             "wildcard": {
-                "value": u"*{0}*".format(value_search_term.lower())
+                _search_field: u"*{0}*".format(_search_term)
             }
         })
 
@@ -302,8 +308,13 @@ def full_text_query(connection, index_name, doc_type, entity_name, sentences, fu
     data = '\n'.join(data)
 
     kwargs = dict(kwargs, body=data, doc_type=doc_type, index=index_name)
-    results = _run_es_search(connection, msearch=True, **kwargs)
-    results = _parse_es_search_results(results.get("responses"))
+    response = None
+    try:
+        response = _run_es_search(connection, msearch=True, **kwargs)
+        results = _parse_es_search_results(response.get("responses"))
+    except Exception as e:
+        raise DataStoreRequestException(f'Error in datastore query on index: {index_name}', engine='elasticsearch',
+                                        request=json.dumps(data), response=json.dumps(response)) from e
     return results
 
 

datastore/exceptions.py

@@ -3,157 +3,112 @@
     'IndexForTransferException', 'AliasForTransferException', 'NonESEngineTransferException',
     'IndexNotFoundException', 'InvalidESURLException', 'SourceDestinationSimilarException',
     'InternalBackupException', 'AliasNotFoundException', 'PointIndexToAliasException',
-    'FetchIndexForAliasException', 'DeleteIndexFromAliasException'
-
+    'FetchIndexForAliasException', 'DeleteIndexFromAliasException', 'DataStoreRequestException'
 ]
 
 
-class DataStoreSettingsImproperlyConfiguredException(Exception):
-    def __init__(self, message=None):
-        self.value = 'Chatbot NER datastore settings are not configured correctly. Please make sure the required' \
-                     ' connection settings are available in the environment variables'
-        if message:
-            self.value = message
-
-    def __str__(self):
-        return repr(self.value)
-
+class BaseDataStoreException(Exception):
+    DEFAULT_ERROR_MESSAGE = None
 
-class EngineNotImplementedException(Exception):
     def __init__(self, message=None):
-        self.value = "Chatbot NER datastore currently supports only the following engines: ['elasticsearch'] . " \
-                     "Please make sure the ENGINE environment variable is correctly set"
+        message = message or self.DEFAULT_ERROR_MESSAGE
         if message:
-            self.value = message
+            super().__init__(message)
+        else:
+            super().__init__()
 
-    def __str__(self):
-        return repr(self.value)
 
+class DataStoreSettingsImproperlyConfiguredException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Chatbot NER datastore settings are not configured correctly. ' \
+                            'Please make sure the required connection settings are available ' \
+                            'in the environment variables'
 
-class EngineConnectionException(Exception):
-    def __init__(self, message=None, engine='the configured engine'):
-        self.value = "Chatbot NER datastore was unable to connect to " + engine + \
-                     ". Please make sure the " + engine + " service is reachable."
-        if message:
-            self.value = message
 
-    def __str__(self):
-        return repr(self.value)
+class EngineNotImplementedException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Chatbot NER datastore currently supports only the following ' \
+                            'engines: ["elasticsearch"]. Please make sure the ENGINE environment ' \
+                            'variable is correctly set'
 
 
-class IndexForTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "ES index has not been configured for transfer. Please configure before transfer."
-        if message:
-            self.value = message
+class EngineConnectionException(BaseDataStoreException):
+    def __init__(self, message=None, engine='configured engine'):
+        if not message:
+            message = 'Chatbot NER datastore was unable to connect to {engine}. ' \
+                      'Please make sure the {engine} service is reachable.'.format(engine=engine)
+        super().__init__(message)
 
-    def __str__(self):
-        return repr(self.value)
 
+class IndexForTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'ES index has not been configured for transfer. Please configure before transfer.'
 
-class AliasForTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "ES alias has not been configured for transfer. Please configure before transfer."
-        if message:
-            self.value = message
 
-    def __str__(self):
-        return repr(self.value)
+class AliasForTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'ES alias has not been configured for transfer. Please configure before transfer.'
 
 
-class NonESEngineTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "Transfer has been triggered for datastore engone other than elastic search"
-        if message:
-            self.value = message
+class NonESEngineTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Transfer has been triggered for datastore engone other than elastic search'
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class IndexNotFoundException(Exception):
+class IndexNotFoundException(BaseDataStoreException):
     """
     This exception will be raised if index is not found in ES
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class InvalidESURLException(Exception):
+class InvalidESURLException(BaseDataStoreException):
     """
     This exception will be raised if the ES URL is invalid
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class SourceDestinationSimilarException(Exception):
+class SourceDestinationSimilarException(BaseDataStoreException):
     """
     This exception will be raised if source is the same as destination
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class InternalBackupException(Exception):
+class InternalBackupException(BaseDataStoreException):
     """
     This exception will be raised for transfer of documents from one index to other within a ES URL
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class AliasNotFoundException(Exception):
+class AliasNotFoundException(BaseDataStoreException):
     """
     This exception will be raised if alias not found in ES
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class PointIndexToAliasException(Exception):
+class PointIndexToAliasException(BaseDataStoreException):
     """
     This exception is raised if the assignment of an index to an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class FetchIndexForAliasException(Exception):
+class FetchIndexForAliasException(BaseDataStoreException):
     """
     This exception is raised if fetch for indices for an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class DeleteIndexFromAliasException(Exception):
+class DeleteIndexFromAliasException(BaseDataStoreException):
     """
     This exception is raised if deletion of an index from an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
+
 
-    def __str__(self):
-        return repr(self.value)
+class DataStoreRequestException(BaseDataStoreException):
+    def __init__(self, message, engine, request, response=None):
+        self.engine = engine
+        self.request = request
+        self.response = response
+        super().__init__(message)

ner_v1/api.py

@@ -5,8 +5,10 @@
 
 import six
 from django.http import HttpResponse
+from elasticsearch import exceptions as es_exceptions
 
 from chatbot_ner.config import ner_logger
+from datastore.exceptions import DataStoreRequestException
 from language_utilities.constant import ENGLISH_LANG
 from ner_constants import (PARAMETER_MESSAGE, PARAMETER_ENTITY_NAME, PARAMETER_STRUCTURED_VALUE,
                            PARAMETER_FALLBACK_VALUE, PARAMETER_BOT_MESSAGE, PARAMETER_TIMEZONE, PARAMETER_REGEX,
@@ -272,9 +274,20 @@ def text(request):
             predetected_values=parameters_dict[PARAMETER_PRIOR_RESULTS]
         )
         ner_logger.debug('Finished %s : %s ' % (parameters_dict[PARAMETER_ENTITY_NAME], entity_output))
-    except TypeError as e:
-        ner_logger.exception('Exception for text_synonym: %s ' % e)
+    except DataStoreRequestException as err:
+        ner_logger.exception(f"Error in requesting ES {request.path}, error: {err}, query: {err.request},"
+                             f" response: {err.response}")
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionTimeout as err:
+        ner_logger.exception(f"Error in text_synonym for: {request.path}, error: {err}")
         return HttpResponse(status=500)
+    except es_exceptions.ConnectionError as err:
+        ner_logger.exception(f"Error in text_synonym for:  {request.path}, error: {err}")
+        return HttpResponse(status=500)
+    except (TypeError, KeyError) as err:
+        ner_logger.exception(f"Error in text_synonym for: {request.path}, error: {err}")
+        return HttpResponse(status=500)
+
     return HttpResponse(json.dumps({'data': entity_output}), content_type='application/json')
 
 
@@ -422,6 +435,15 @@ def city(request):
     except TypeError as e:
         ner_logger.exception('Exception for city: %s ' % e)
         return HttpResponse(status=500)
+    except KeyError as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionTimeout as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionError as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
 
     return HttpResponse(json.dumps({'data': entity_output}), content_type='application/json')
 

ner_v2/api.py

@@ -6,8 +6,10 @@
 import six
 from django.http import HttpResponse, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
+from elasticsearch import exceptions as es_exceptions
 
 from chatbot_ner.config import ner_logger
+from datastore.exceptions import DataStoreRequestException
 from language_utilities.constant import ENGLISH_LANG
 from ner_constants import PARAMETER_MESSAGE, PARAMETER_ENTITY_NAME, PARAMETER_STRUCTURED_VALUE, \
     PARAMETER_FALLBACK_VALUE, \
@@ -19,7 +21,7 @@
 from ner_v2.detectors.pattern.phone_number.phone_number_detection import PhoneDetector
 from ner_v2.detectors.temporal.date.date_detection import DateAdvancedDetector
 from ner_v2.detectors.temporal.time.time_detection import TimeDetector
-from ner_v2.detectors.textual.utils import get_text_entity_detection_data, verify_text_request
+from ner_v2.detectors.textual.utils import get_text_entity_detection_data, validate_text_request, InvalidTextRequest
 
 
 def get_parameters_dictionary(request):
@@ -674,22 +676,33 @@ def text(request):
         ner_logger.debug("Fetching result")
 
         try:
-            verify_text_request(request)
+            validate_text_request(request)
             # if verify success get detection data
             data = get_text_entity_detection_data(request)
-
-        except KeyError as err:
+        except InvalidTextRequest as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Error in validating request body for {request.path}, error: {err}")
+            return JsonResponse(response, status=400)
+        except DataStoreRequestException as err:
             response = {"success": False, "error": str(err)}
-            # TODO: move to ner_logger.error
-            ner_logger.exception(response)
+            ner_logger.exception(f"Error in requesting ES {request.path}, error: {err}, query: {err.request},"
+                                 f" response: {err.response}")
             return JsonResponse(response, status=400)
-        except TypeError as err:
+        except es_exceptions.ConnectionTimeout as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Connection timed out for ES   {request.path}, error: {err}")
+            return JsonResponse(response, status=500)
+        except es_exceptions.ConnectionError as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Error in connection to ES {request.path}, error: {err}")
+            return JsonResponse(response, status=500)
+        except (TypeError, KeyError) as err:
             response = {"success": False, "error": str(err)}
-            ner_logger.exception(response)
+            ner_logger.exception(f"Error in validating type {request.path}, error: {err}")
             return JsonResponse(response, status=400)
         except Exception as err:
             response = {"success": False, "error": str(err)}
-            ner_logger.exception(response)
+            ner_logger.exception(f"General exception for {request.path}, error: {err}")
             return JsonResponse(response, status=500)
     if data:
         response = {"success": True, "error": None, "data": data}

ner_v2/detectors/pattern/phone_number/phone_number_detection.py

@@ -1,12 +1,16 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-from ner_v2.detectors.base_detector import BaseDetector
-from ner_v2.detectors.numeral.number.number_detection import NumberDetector
-from language_utilities.constant import ENGLISH_LANG
+
 import re
+
 import phonenumbers
+import regex
 from six.moves import zip
 
+from language_utilities.constant import ENGLISH_LANG
+from ner_v2.detectors.base_detector import BaseDetector
+from ner_v2.detectors.numeral.number.number_detection import NumberDetector
+
 
 class PhoneDetector(BaseDetector):
     """
@@ -51,6 +55,8 @@ def get_country_code_from_locale(self):
         This method sets self.country_code from given locale
         """
         regex_pattern = re.compile('[-_](.*$)', re.U)
+        self.locale = regex.sub("\\p{Pd}", "-",
+                                self.locale)  # This will replace all types of dashes(em or en) by hyphen.
         match = regex_pattern.findall(self.locale)
         if match:
             return match[0].upper()