Remove twitter storage adapter in favor of trainer

chatterbot/adapters/storage/__init__.py

@@ -2,4 +2,3 @@
 from .django_storage import DjangoStorageAdapter
 from .jsonfile import JsonFileStorageAdapter
 from .mongodb import MongoDatabaseAdapter
-from .twitter_storage import TwitterAdapter

chatterbot/chatterbot.py

@@ -70,7 +70,7 @@ def __init__(self, name, **kwargs):
         # Use specified trainer or fall back to the default
         trainer = kwargs.get('trainer', 'chatterbot.trainers.Trainer')
         TrainerClass = import_module(trainer)
-        self.trainer = TrainerClass(self.storage)
+        self.trainer = TrainerClass(self.storage, **kwargs)
 
         self.logger = kwargs.get('logger', logging.getLogger(__name__))
 

chatterbot/trainers.py

@@ -1,12 +1,14 @@
 from .conversation import Statement, Response
 from .corpus import Corpus
+import logging
 
 
 class Trainer(object):
 
     def __init__(self, storage, **kwargs):
         self.storage = storage
         self.corpus = Corpus()
+        self.logger = logging.getLogger(__name__)
 
     def train(self, *args, **kwargs):
         raise self.TrainerInitializationException()
@@ -82,3 +84,89 @@ def train(self, *corpora):
             for data in corpus_data:
                 for pair in data:
                     trainer.train(pair)
+
+
+class TwitterTrainer(Trainer):
+
+    def __init__(self, storage, **kwargs):
+        super(TwitterTrainer, self).__init__(storage, **kwargs)
+        from twitter import Api as TwitterApi
+
+        self.api = TwitterApi(
+            consumer_key=kwargs.get('twitter_consumer_key'),
+            consumer_secret=kwargs.get('twitter_consumer_secret'),
+            access_token_key=kwargs.get('twitter_access_token_key'),
+            access_token_secret=kwargs.get('twitter_access_token_secret')
+        )
+
+    def random_word(self, base_word='random'):
+        """
+        Generate a random word using the Twitter API.
+
+        Search twitter for recent tweets containing the term 'random'.
+        Then randomly select one word from those tweets and do another
+        search with that word. Return a randomly selected word from the
+        new set of results.
+        """
+        import random
+        random_tweets = self.api.GetSearch(term=base_word, count=5)
+        random_words = self.get_words_from_tweets(random_tweets)
+        random_word = random.choice(list(random_words))
+        tweets = self.api.GetSearch(term=random_word, count=5)
+        words = self.get_words_from_tweets(tweets)
+        word = random.choice(list(words))
+        return word
+
+    def get_words_from_tweets(self, tweets):
+        """
+        Given a list of tweets, return the set of
+        words from the tweets.
+        """
+        words = set()
+
+        for tweet in tweets:
+            # TODO: Handle non-ascii characters properly
+            cleaned_text = ''.join(
+                [i if ord(i) < 128 else ' ' for i in tweet.text]
+            )
+            tweet_words = cleaned_text.split()
+
+            for word in tweet_words:
+                # If the word contains only letters with a length from 4 to 9
+                if word.isalpha() and len(word) > 3 and len(word) <= 9:
+                    words.add(word)
+
+        return words
+
+    def get_statements(self):
+        """
+        Returns list of random statements from the API.
+        """
+        from twitter import TwitterError
+        statements = []
+
+        # Generate a random word
+        random_word = self.random_word()
+
+        self.logger.info(u'Requesting 50 random tweets containing the word {}'.format(random_word))
+        tweets = self.api.GetSearch(term=random_word, count=50)
+        for tweet in tweets:
+            statement = Statement(tweet.text)
+
+            if tweet.in_reply_to_status_id:
+                try:
+                    status = self.api.GetStatus(tweet.in_reply_to_status_id)
+                    statement.add_response(Response(status.text))
+                    statements.append(statement)
+                except TwitterError as e:
+                    self.logger.warning(str(e))
+
+        self.logger.info('Adding {} tweets with responses'.format(len(statements)))
+
+        return statements
+
+    def train(self):
+        for i in range(0, 10):
+            statements = self.get_statements()
+            for statement in statements:
+                self.storage.update(statement, force=True)

docs/adapters/storage.rst

@@ -66,30 +66,3 @@ can set the `database_uri` parameter to the uri of your database.
 .. code-block:: python
 
    database_uri='mongodb://example.com:8100/'
-
-Twitter Adapter
-==============================
-
-.. autofunction:: chatterbot.adapters.storage.TwitterAdapter
-
-"chatterbot.adapters.storage.TwitterAdapter"
-
-Create an app from you twiter acccount, Once created
-It will have following app credentails that are required to work with 
-TwitterAdapter.
-
-twitter_consumer_key
---------------------
-Consumer key of twitter app.
-
-twitter_consumer_secret
------------------------
-Consumer secret of twitter app.
-
-twitter_access_token_key
-------------------------
-Access token key of twitter app.
-
-twitter_access_token_secret
----------------------------
-Access token secret of twitter app.

docs/training.rst

@@ -15,16 +15,20 @@ The case that someone wants to create a custom training module typically comes u
 
 ..  _set_trainer:
 
+
 Setting the training class
 ==========================
 
 ChatterBot comes with training classes built in, or you can create your own
 if needed. To use a training class you must import it and pass it to
 the `set_trainer()` method before calling `train()`.
 
+
 Training via list data
 ======================
 
+.. autofunction:: chatterbot.trainers.ListTrainer
+
 For the training, process, you will need to pass in a list of statements where the order of each statement is based on it's placement in a given conversation.
 
 For example, if you were to run bot of the following training calls, then the resulting chatterbot would respond to both statements of "Hi there!" and "Greetings!" by saying "Hello".
@@ -59,9 +63,12 @@ This will establish each item in the list as a possible response to it's predece
        "You are welcome.",
    ])
 
+
 Training with corpus data
 =========================
 
+.. autofunction:: chatterbot.trainers.ChatterBotCorpusTrainer
+
 ChatterBot comes with a corpus data and utility module that makes it easy to
 quickly train your bot to communicate. To do so, simply specify the corpus
 data modules you want to use.
@@ -91,6 +98,35 @@ conversations corpora then you would simply specify them.
        "chatterbot.corpus.english.conversations"
    )
 
+
+Training with the Twitter API
+=============================
+
+.. autofunction:: chatterbot.trainers.TwitterTrainer
+
+Create an new app using you twiter acccount. Once created,
+it will provide you with the following credentails that are
+required to work with the Twitter API.
+
++-------------------------------------+-------------------------------------+
+| Parameter                           | Description                         | 
++=====================================+=====================================+
+| :code:`twitter_consumer_key`        | Consumer key of twitter app.        |
++-------------------------------------+-------------------------------------+
+| :code:`twitter_consumer_secret`     | Consumer secret of twitter app.     | 
++-------------------------------------+-------------------------------------+
+| :code:`twitter_access_token_key`    | Access token key of twitter app.    | 
++-------------------------------------+-------------------------------------+
+| :code:`twitter_access_token_secret` | Access token secret of twitter app. | 
++-------------------------------------+-------------------------------------+
+
+Twitter training example
+------------------------
+
+.. literalinclude:: ../examples/twitter_training_example.py
+   :language: python
+
+
 Creating a new training class
 =============================
 
@@ -105,6 +141,7 @@ parameters you choose.
 
 Take a look at the existing `trainer classes on GitHub`_ for examples.
 
+
 The ChatterBot Corpus
 =====================
 
@@ -122,6 +159,7 @@ To explore what languages and sets of corpora are available, check out the `chat
    If you are interested in contributing a new language corpus, or adding content to an existing language in the corpus,
    please feel free to submit a pull request on ChatterBot's GitHub page. Contributions are welcomed!
 
+
 Exporting your chat bot's database as a training corpus
 =======================================================