Support for better handling error from ES search

datastore/elastic_search/query.py

@@ -12,6 +12,7 @@
 from six.moves import zip
 
 from datastore import constants
+from datastore.exceptions import DataStoreRequestException
 from external_api.constants import SENTENCE, ENTITIES
 from language_utilities.constant import ENGLISH_LANG
 from lib.nlp.const import TOKENIZER
@@ -336,6 +337,9 @@ def _run_es_search(connection, msearch=False, **kwargs):
         raise ValueError('Scrolling is not supported in msearch mode')
 
     result = connection.search(scroll=scroll, **kwargs)
+    if 'hits' not in result:
+        raise DataStoreRequestException(f'No hits from ES search for kwargs -> {kwargs}')
+
     scroll_id = result['_scroll_id']
     scroll_size = result['hits']['total']
     hit_list = result['hits']['hits']

datastore/exceptions.py

@@ -3,157 +3,108 @@
     'IndexForTransferException', 'AliasForTransferException', 'NonESEngineTransferException',
     'IndexNotFoundException', 'InvalidESURLException', 'SourceDestinationSimilarException',
     'InternalBackupException', 'AliasNotFoundException', 'PointIndexToAliasException',
-    'FetchIndexForAliasException', 'DeleteIndexFromAliasException'
-
+    'FetchIndexForAliasException', 'DeleteIndexFromAliasException', 'DataStoreRequestException'
 ]
 
 
-class DataStoreSettingsImproperlyConfiguredException(Exception):
-    def __init__(self, message=None):
-        self.value = 'Chatbot NER datastore settings are not configured correctly. Please make sure the required' \
-                     ' connection settings are available in the environment variables'
-        if message:
-            self.value = message
-
-    def __str__(self):
-        return repr(self.value)
-
+class BaseDataStoreException(Exception):
+    DEFAULT_ERROR_MESSAGE = None
 
-class EngineNotImplementedException(Exception):
     def __init__(self, message=None):
-        self.value = "Chatbot NER datastore currently supports only the following engines: ['elasticsearch'] . " \
-                     "Please make sure the ENGINE environment variable is correctly set"
+        message = message or self.DEFAULT_ERROR_MESSAGE
         if message:
-            self.value = message
+            super().__init__(message)
+        else:
+            super().__init__()
 
-    def __str__(self):
-        return repr(self.value)
 
+class DataStoreSettingsImproperlyConfiguredException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Chatbot NER datastore settings are not configured correctly. ' \
+                            'Please make sure the required connection settings are available ' \
+                            'in the environment variables'
 
-class EngineConnectionException(Exception):
-    def __init__(self, message=None, engine='the configured engine'):
-        self.value = "Chatbot NER datastore was unable to connect to " + engine + \
-                     ". Please make sure the " + engine + " service is reachable."
-        if message:
-            self.value = message
 
-    def __str__(self):
-        return repr(self.value)
+class EngineNotImplementedException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Chatbot NER datastore currently supports only the following ' \
+                            'engines: ["elasticsearch"]. Please make sure the ENGINE environment ' \
+                            'variable is correctly set'
 
 
-class IndexForTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "ES index has not been configured for transfer. Please configure before transfer."
-        if message:
-            self.value = message
+class EngineConnectionException(BaseDataStoreException):
+    def __init__(self, message=None, engine='configured engine'):
+        if not message:
+            message = 'Chatbot NER datastore was unable to connect to {engine}. ' \
+                      'Please make sure the {engine} service is reachable.'.format(engine=engine)
+        super().__init__(message)
 
-    def __str__(self):
-        return repr(self.value)
 
+class IndexForTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'ES index has not been configured for transfer. Please configure before transfer.'
 
-class AliasForTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "ES alias has not been configured for transfer. Please configure before transfer."
-        if message:
-            self.value = message
 
-    def __str__(self):
-        return repr(self.value)
+class AliasForTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'ES alias has not been configured for transfer. Please configure before transfer.'
 
 
-class NonESEngineTransferException(Exception):
-    def __init__(self, message=None):
-        self.value = "Transfer has been triggered for datastore engone other than elastic search"
-        if message:
-            self.value = message
+class NonESEngineTransferException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'Transfer has been triggered for datastore engone other than elastic search'
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class IndexNotFoundException(Exception):
+class IndexNotFoundException(BaseDataStoreException):
     """
     This exception will be raised if index is not found in ES
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class InvalidESURLException(Exception):
+class InvalidESURLException(BaseDataStoreException):
     """
     This exception will be raised if the ES URL is invalid
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class SourceDestinationSimilarException(Exception):
+class SourceDestinationSimilarException(BaseDataStoreException):
     """
     This exception will be raised if source is the same as destination
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class InternalBackupException(Exception):
+class InternalBackupException(BaseDataStoreException):
     """
     This exception will be raised for transfer of documents from one index to other within a ES URL
     """
-    def __init__(self, message=None):
-        self.value = message
-
-    def __str__(self):
-        return repr(self.value)
+    pass
 
 
-class AliasNotFoundException(Exception):
+class AliasNotFoundException(BaseDataStoreException):
     """
     This exception will be raised if alias not found in ES
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class PointIndexToAliasException(Exception):
+class PointIndexToAliasException(BaseDataStoreException):
     """
     This exception is raised if the assignment of an index to an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class FetchIndexForAliasException(Exception):
+class FetchIndexForAliasException(BaseDataStoreException):
     """
     This exception is raised if fetch for indices for an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
 
-    def __str__(self):
-        return repr(self.value)
 
-
-class DeleteIndexFromAliasException(Exception):
+class DeleteIndexFromAliasException(BaseDataStoreException):
     """
     This exception is raised if deletion of an index from an alias fails
     """
-    def __init__(self, message=None):
-        self.value = message
+    pass
+
 
-    def __str__(self):
-        return repr(self.value)
+class DataStoreRequestException(BaseDataStoreException):
+    DEFAULT_ERROR_MESSAGE = 'ES request causes an exception or no results'

ner_v1/api.py

@@ -5,8 +5,10 @@
 
 import six
 from django.http import HttpResponse
+from elasticsearch import exceptions as es_exceptions
 
 from chatbot_ner.config import ner_logger
+from datastore.exceptions import DataStoreRequestException
 from language_utilities.constant import ENGLISH_LANG
 from ner_constants import (PARAMETER_MESSAGE, PARAMETER_ENTITY_NAME, PARAMETER_STRUCTURED_VALUE,
                            PARAMETER_FALLBACK_VALUE, PARAMETER_BOT_MESSAGE, PARAMETER_TIMEZONE, PARAMETER_REGEX,
@@ -272,9 +274,22 @@ def text(request):
             predetected_values=parameters_dict[PARAMETER_PRIOR_RESULTS]
         )
         ner_logger.debug('Finished %s : %s ' % (parameters_dict[PARAMETER_ENTITY_NAME], entity_output))
-    except TypeError as e:
-        ner_logger.exception('Exception for text_synonym: %s ' % e)
+    except DataStoreRequestException as err:
+        ner_logger.exception(f"Error in text_synonym for requesting ES {request.path}, error: {err}")
+        return HttpResponse(status=500)
+    except TypeError as err:
+        ner_logger.exception(f"Error in text_synonym for: {request.path}, error: {err}")
+        return HttpResponse(status=500)
+    except KeyError as err:
+        ner_logger.exception(f"Error in text_synonym for: {request.path}, error: {err}")
         return HttpResponse(status=500)
+    except es_exceptions.ConnectionTimeout as err:
+        ner_logger.exception(f"Error in text_synonym for: {request.path}, error: {err}")
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionError as err:
+        ner_logger.exception(f"Error in text_synonym for:  {request.path}, error: {err}")
+        return HttpResponse(status=500)
+
     return HttpResponse(json.dumps({'data': entity_output}), content_type='application/json')
 
 
@@ -422,6 +437,15 @@ def city(request):
     except TypeError as e:
         ner_logger.exception('Exception for city: %s ' % e)
         return HttpResponse(status=500)
+    except KeyError as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionTimeout as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
+    except es_exceptions.ConnectionError as e:
+        ner_logger.exception('Exception for text_synonym: %s ' % e)
+        return HttpResponse(status=500)
 
     return HttpResponse(json.dumps({'data': entity_output}), content_type='application/json')
 

ner_v2/api.py

@@ -6,8 +6,10 @@
 import six
 from django.http import HttpResponse, JsonResponse
 from django.views.decorators.csrf import csrf_exempt
+from elasticsearch import exceptions as es_exceptions
 
 from chatbot_ner.config import ner_logger
+from datastore.exceptions import DataStoreRequestException
 from language_utilities.constant import ENGLISH_LANG
 from ner_constants import PARAMETER_MESSAGE, PARAMETER_ENTITY_NAME, PARAMETER_STRUCTURED_VALUE, \
     PARAMETER_FALLBACK_VALUE, \
@@ -19,7 +21,7 @@
 from ner_v2.detectors.pattern.phone_number.phone_number_detection import PhoneDetector
 from ner_v2.detectors.temporal.date.date_detection import DateAdvancedDetector
 from ner_v2.detectors.temporal.time.time_detection import TimeDetector
-from ner_v2.detectors.textual.utils import get_text_entity_detection_data, verify_text_request
+from ner_v2.detectors.textual.utils import get_text_entity_detection_data, validate_text_request, InvalidTextRequest
 
 
 def get_parameters_dictionary(request):
@@ -674,22 +676,32 @@ def text(request):
         ner_logger.debug("Fetching result")
 
         try:
-            verify_text_request(request)
+            validate_text_request(request)
             # if verify success get detection data
             data = get_text_entity_detection_data(request)
-
-        except KeyError as err:
+        except InvalidTextRequest as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Error in validating request body for {request.path}, error: {err}")
+            return JsonResponse(response, status=400)
+        except DataStoreRequestException as err:
             response = {"success": False, "error": str(err)}
-            # TODO: move to ner_logger.error
-            ner_logger.exception(response)
+            ner_logger.exception(f"Error in requesting ES {request.path}, error: {err}")
             return JsonResponse(response, status=400)
         except TypeError as err:
             response = {"success": False, "error": str(err)}
-            ner_logger.exception(response)
+            ner_logger.exception(f"Error in validating type {request.path}, error: {err}")
             return JsonResponse(response, status=400)
+        except es_exceptions.ConnectionTimeout as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Connection timed out for ES   {request.path}, error: {err}")
+            return JsonResponse(response, status=500)
+        except es_exceptions.ConnectionError as err:
+            response = {"success": False, "error": str(err)}
+            ner_logger.exception(f"Error in connection to ES {request.path}, error: {err}")
+            return JsonResponse(response, status=500)
         except Exception as err:
             response = {"success": False, "error": str(err)}
-            ner_logger.exception(response)
+            ner_logger.exception(f"General exception for {request.path}, error: {err}")
             return JsonResponse(response, status=500)
     if data:
         response = {"success": True, "error": None, "data": data}

ner_v2/detectors/textual/queries.py

@@ -8,6 +8,7 @@
 from six import string_types
 
 from datastore import constants
+from datastore.exceptions import DataStoreRequestException
 from language_utilities.constant import ENGLISH_LANG
 from lib.nlp.const import TOKENIZER
 
@@ -199,7 +200,8 @@ def _parse_multi_entity_es_results(results_list):
         for results in results_list:
             entity_dict = {}
             entity_variants_to_values_dict = {}
-
+            if 'hits' not in results:
+                raise DataStoreRequestException(f'No hits from ES search for results -> {results}')
             if results['hits']['total'] > 0:
                 for hit in results['hits']['hits']:
                     if 'highlight' not in hit: