Merge develop to master

chatbot_ner/config.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import logging.handlers
 import os
 

chatbot_ner/settings.py

@@ -9,6 +9,7 @@
 """
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+from __future__ import absolute_import
 import os
 import sys
 

chatbot_ner/wsgi.py

@@ -6,7 +6,7 @@
 For more information on this file, see
 https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
 """
-
+from __future__ import absolute_import
 import os
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "chatbot_ner.settings")
 

datastore/commands/create_datastore.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from django.core.management.base import BaseCommand
 from datastore import DataStore
 

datastore/commands/delete_datastore.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from django.core.management.base import BaseCommand
 from datastore import DataStore
 

datastore/commands/delete_entity_data_datastore.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from django.core.management.base import BaseCommand
 from datastore import DataStore
 

datastore/commands/populate_datastore.py

@@ -1,7 +1,6 @@
+from __future__ import absolute_import
 import os
-
 from django.core.management.base import BaseCommand
-
 from datastore import DataStore
 
 

datastore/commands/repopulate_datastore.py

@@ -1,7 +1,6 @@
+from __future__ import absolute_import
 import os
-
 from django.core.management.base import BaseCommand
-
 from datastore import DataStore
 
 

datastore/constants.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import elasticsearch
 import os
 from chatbot_ner.settings import BASE_DIR

datastore/datastore.py

@@ -10,9 +10,10 @@
 from datastore.exceptions import (DataStoreSettingsImproperlyConfiguredException, EngineNotImplementedException,
                                   EngineConnectionException, NonESEngineTransferException, IndexNotFoundException)
 from lib.singleton import Singleton
+import six
 
 
-class DataStore(object):
+class DataStore(six.with_metaclass(Singleton, object)):
     """
     Singleton class to connect to engine storing entity related data
 
@@ -34,7 +35,6 @@ class DataStore(object):
         _store_name: Name of the database/index to query on the engine server
         _client_or_connection: Low level connection object to the engine, None at initialization
     """
-    __metaclass__ = Singleton
 
     def __init__(self):
         """

datastore/elastic_search/__init__.py

@@ -1 +1,2 @@
+from __future__ import absolute_import
 from . import connect, create, populate, query, transfer

datastore/elastic_search/connect.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from elasticsearch import Elasticsearch
 from chatbot_ner.config import CHATBOT_NER_DATASTORE
 from datastore.elastic_search.transfer import ESTransfer

datastore/elastic_search/transfer.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import requests
 import json
 from elasticsearch import Elasticsearch, RequestsHttpConnection

datastore/utils.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import csv
 import os
 from collections import defaultdict

external_api/api.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import json
 from django.http import HttpResponse
 from datastore.datastore import DataStore

initial_setup.py

@@ -1,6 +1,6 @@
+from __future__ import absolute_import
 import os
 import time
-
 import nltk
 
 BASE_DIR = os.path.dirname(__file__)

language_utilities/utils.py

@@ -2,7 +2,9 @@
 from language_utilities.constant import ENGLISH_LANG
 from language_utilities.constant import TRANSLATED_TEXT
 from chatbot_ner.config import ner_logger, GOOGLE_TRANSLATE_API_KEY
-import urllib
+import six.moves.urllib.request
+import six.moves.urllib.parse
+import six.moves.urllib.error
 import requests
 
 
@@ -18,8 +20,16 @@ def unicode_urlencode(params):
         (str): url string with params encoded
     """
     if isinstance(params, dict):
-        params = params.items()
-    return urllib.urlencode([(k, isinstance(v, unicode) and v.encode('utf-8') or v) for k, v in params])
+        params = list(params.items())
+    return six.moves.urllib.parse.urlencode(
+        [(
+            k, isinstance(
+                v,
+                six.text_type
+            ) and v.encode('utf-8') or v
+        ) for (k, v) in params
+        ]
+    )
 
 
 def translate_text(text, source_language_code, target_language_code=ENGLISH_LANG):

lib/aws_utils.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import boto
 import boto3
 from chatbot_ner.config import ner_logger

lib/nlp/const.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import os
 from lib.nlp.etc import store_data_in_list
 from lib.nlp.ngram import Ngram

lib/nlp/etc.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import csv
 
 
@@ -36,10 +37,10 @@ def filter_list(list_to_filter, remove_elements):
     Returns:
         list of str: The list of elements in first list but not in second
 
-        For example output = remove_one_list_to_another(list1=['hi', 'hello', 'bye'],['hi', 'bye'])
-        print output
+        For example, output = filter_list(['hi', 'hello', 'bye'],['hi', 'bye'])
+        print(output)
         >> ['hello']
     """
-    return filter(lambda token: token not in remove_elements, list_to_filter)
+    return [item for item in list_to_filter if item not in remove_elements]
 
 

lib/nlp/lemmatizer.py

@@ -1,12 +1,14 @@
+from __future__ import absolute_import
 from nltk.stem import WordNetLemmatizer
 
 # constants
 from lib.singleton import Singleton
+import six
 
 WORDNET_LEMMATIZER = 'WORDNET_LEMMATIZER'
 
 
-class Lemmatizer(object):
+class Lemmatizer(six.with_metaclass(Singleton, object)):
     """
     Detects the lemma of a word.
     Its a wrapper class which can be used to call respective lemmatizer library. Currently, It has been integrated
@@ -41,8 +43,6 @@ class Lemmatizer(object):
 
     """
 
-    __metaclass__ = Singleton
-
     def __init__(self, lemmatizer_selected=WORDNET_LEMMATIZER):
         """Initializes a Lemmatizer object
 

lib/nlp/ngram.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import nltk
 
 
@@ -39,7 +40,7 @@ def ngram_list(n, word_list, stop_word_list=None):
         all_ngrams = nltk.ngrams(word_list, n)
         ngram_list = []
         for ngram in all_ngrams:
-            lowered_ngram_tokens = map(lambda token: token.lower(), ngram)
+            lowered_ngram_tokens = [token.lower() for token in ngram]
             if any(token not in stop_word_set for token in lowered_ngram_tokens):
                 ngram_list.append(' '.join(ngram))
         return ngram_list

lib/nlp/pos.py

@@ -1,7 +1,9 @@
+from __future__ import absolute_import
 import nltk
 
 # constants
 from lib.singleton import Singleton
+import six
 
 DEFAULT_NLTK_TAGGER_PATH = 'taggers/maxent_treebank_pos_tagger/english.pickle'
 NLTK_MAXENT_TAGGER = 'NLTK_MAXENT_TAGGER'
@@ -18,8 +20,7 @@ def tag(self, tokens, tagset=None):
         return tagged_tokens
 
 
-class POS(object):
-    __metaclass__ = Singleton
+class POS(six.with_metaclass(Singleton, object)):
 
     def __init__(self, tagger_selected=NLTK_AP_TAGGER, tagger=None):
         self.tagger_dict = {

lib/nlp/regexreplace.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import re
 
 

lib/nlp/stemmer.py

@@ -1,12 +1,14 @@
+from __future__ import absolute_import
 from nltk.stem.porter import PorterStemmer
 
 # constants
 from lib.singleton import Singleton
+import six
 
 PORTER_STEMMER = 'PORTER_STEMMER'
 
 
-class Stemmer(object):
+class Stemmer(six.with_metaclass(Singleton, object)):
     """
     Detects the stemmer of a word.
     Its a wrapper class which can be used to call respective stemmer library. Currently, It has been integrated
@@ -35,8 +37,6 @@ class Stemmer(object):
 
     """
 
-    __metaclass__ = Singleton
-
     def __init__(self, stemmer_selected=PORTER_STEMMER):
         """Initializes a Stemmer object
 

lib/nlp/tokenizer.py

@@ -1,18 +1,19 @@
+from __future__ import absolute_import
 import re
-
 import nltk
 import regex
 
 # constants
 from lib.singleton import Singleton
+import six
 
 NLTK_TOKENIZER = 'WORD_TOKENIZER'
 PRELOADED_NLTK_TOKENIZER = 'PRELOADED_NLTK_TOKENIZER'
 LUCENE_STANDARD_TOKENIZER = 'LUCENE_STANDARD_TOKENIZER'
 WHITESPACE_TOKENIZER = 'WHITESPACE_TOKENIZER'
 
 
-class Tokenizer(object):
+class Tokenizer(six.with_metaclass(Singleton, object)):
     """
     Returns the list of the tokens from the text
     Its a wrapper class which can be used to call respective tokenizer library. Currently, It has been integrated
@@ -31,8 +32,6 @@ class Tokenizer(object):
 
     """
 
-    __metaclass__ = Singleton
-
     def __init__(self, tokenizer_selected=NLTK_TOKENIZER):
         """Initializes a Tokenizer object
 

manage.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import absolute_import
 import os
 import sys
 

models/crf/models.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from chatbot_ner.config import CITY_MODEL_TYPE, DATE_MODEL_TYPE
 from .constant import CRF_MODEL_TYPE, CITY_ENTITY_TYPE, DATE_ENTITY_TYPE
 from .test import PredictCRF

models/crf/output_generation/city.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import models.crf.constant as model_constant
 
 

models/crf/output_generation/date.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import models.crf.constant as model_constant
 
 

models/crf/test.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 from chatbot_ner.config import ner_logger, CITY_MODEL_PATH, DATE_MODEL_PATH
 from lib.nlp.const import nltk_tokenizer
 from lib.nlp.pos import POS

models/crf_v2/crf_detect_entity.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from lib.nlp.tokenizer import Tokenizer, NLTK_TOKENIZER
 from .crf_preprocess_data import CrfPreprocessData
 from .get_crf_tagger import CrfModel

models/crf_v2/crf_preprocess_data.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import numpy as np
 from lib.nlp.pos import POS
 import re

models/crf_v2/crf_train.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import pycrfsuite
 from chatbot_ner.config import ner_logger, CRF_MODEL_S3_BUCKET_NAME, CRF_MODEL_S3_BUCKET_REGION, CRF_MODELS_PATH
 from datastore.datastore import DataStore

models/crf_v2/get_crf_tagger.py

@@ -1,14 +1,15 @@
+from __future__ import absolute_import
 from lib.singleton import Singleton
 from chatbot_ner.config import CRF_MODEL_S3_BUCKET_REGION, CRF_MODEL_S3_BUCKET_NAME, ner_logger
 from lib.aws_utils import read_model_dict_from_s3
 import pycrfsuite
+import six
 
 
-class CrfModel(object):
+class CrfModel(six.with_metaclass(Singleton, object)):
     """
     This class is used to load the crf tagger for a given entity
     """
-    __metaclass__ = Singleton
 
     def __init__(self, entity_name):
         """

models/crf_v2/load_word_embeddings.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import numpy as np
 from lib.singleton import Singleton
 import pickle
@@ -6,13 +7,13 @@
 import json
 from models.crf_v2.constants import TEXT_LIST, CRF_WORD_EMBEDDINGS_LIST
 from chatbot_ner.config import ner_logger
+import six
 
 
-class LoadWordEmbeddings(object):
+class LoadWordEmbeddings(six.with_metaclass(Singleton, object)):
     """
     This method is used to load the word_embeddings into the memory
     """
-    __metaclass__ = Singleton
 
     def __init__(self):
         """

ner_v1/chatbot/combine_detection_logic.py

@@ -1,7 +1,6 @@
+from __future__ import absolute_import
 from collections import defaultdict
-
 from six import iteritems
-
 from lib.nlp.const import TOKENIZER
 from lib.nlp.regexreplace import RegexReplace
 from ner_constants import ORIGINAL_TEXT, DETECTION_METHOD, FROM_MESSAGE, FROM_MODEL_VERIFIED, FROM_MODEL_NOT_VERIFIED

ner_v1/chatbot/entity_detection.py

@@ -1077,7 +1077,7 @@ def output_entity_dict_list(entity_value_list, original_text_list, detection_met
 
     entity_list = []
     for i, entity_value in enumerate(entity_value_list):
-        if type(entity_value) in [str, unicode]:
+        if type(entity_value) in [str, six.text_type]:
             entity_value = {
                 ENTITY_VALUE_DICT_KEY: entity_value
             }

ner_v1/chatbot/tag_message.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from ner_v1.chatbot.combine_detection_logic import combine_output_of_detection_logic_and_tag
 from ner_v1.chatbot.entity_detection import get_text, get_city, get_date, get_time, get_email, \
     get_phone_number, get_budget, get_number, get_pnr, get_shopping_size