Merge pull request #11 from scarlehoff/facebook_backend

Implementation of a Facebook backend for pybliotecario

pybliotecario.ini.example

@@ -3,6 +3,12 @@ TOKEN = <telegram bot token>
 chat_id = <chat_id number> 
 main_folder = /home/<username>/.pybliotecario
 
+
+[FACEBOOK]
+verify = <verify token from fb>
+app_token = <app token from fb>
+chat_id = <chat id> # optional
+
 # Other configs
 [ARXIV]
 arxiv_filter_dict = {'title' : ["Higgs"], 'summary' : ["VBF"]}

setup.py

@@ -37,6 +37,7 @@
         "psutil",
         "wikipedia",
     ],
+    extras_require={"facebook": ["flask", "requests_toolbelt"]},
     entry_points={
         "console_scripts": [
             "{0} = pybliotecario.pybliotecario:main".format(pybliotecario_name),

src/pybliotecario/backend/__init__.py

@@ -1,2 +1,3 @@
 from .telegram_util import TelegramUtil
 from .backend_test import TestUtil
+from .facebook_util import FacebookUtil

src/pybliotecario/backend/basic_backend.py

@@ -9,6 +9,7 @@
 
 from abc import ABC, abstractmethod, abstractproperty
 import logging
+import urllib
 import json
 
 logger = logging.getLogger(__name__)
@@ -26,16 +27,15 @@ class Message(ABC):
     _type = "Abstract"
     _original = None
 
-    _message_dict = {
-        "chat_id": None,
-        "username": None,
-        "command": None,
-        "file_id": None,
-        "text": None,
-        "ignore": False,
-    }
-
     def __init__(self, update):
+        self._message_dict = {
+            "chat_id": None,
+            "username": None,
+            "command": None,
+            "file_id": None,
+            "text": None,
+            "ignore": False,
+        }
         self._original = update
         self._parse_update(update)
         # After the information is parsed, log the message!
@@ -44,6 +44,22 @@ def __init__(self, update):
     def __str__(self):
         return json.dumps(self._message_dict)
 
+    def _parse_command(self, text):
+        """ Parse any msg starting with / """
+        separate_command = text.split(" ", 1)
+        # Remove the / from the command
+        command = separate_command[0][1:]
+        # Absorb the @ in case it is a directed command!
+        if "@" in command:
+            command = command.split("@")[0]
+        # Check whether the command comes alone or has arguments
+        if len(separate_command) == 1:
+            text = ""
+        else:
+            text = separate_command[1]
+        self._message_dict["command"] = command
+        self._message_dict["text"] = text
+
     @abstractmethod
     def _parse_update(self, update):
         """ Parse the update and fill in _message_dict """
@@ -133,7 +149,7 @@ def _message_class(self):
     def act_on_updates(self, action_function, not_empty=False):
         """
         Receive the input using _get_updates, parse it with
-        the telegram message class and act in consequence
+        the message class and act in consequence
         """
         all_updates = self._get_updates(not_empty=not_empty)
         for update in all_updates:
@@ -142,12 +158,14 @@ def act_on_updates(self, action_function, not_empty=False):
 
     def send_image(self, img_path, chat):
         """ Sends an image """
-        logger.error("This backend does not implement sending files")
+        logger.error("This backend does not implement sending images")
 
     def send_file(self, filepath, chat):
         """ Sends a file """
         logger.error("This backend does not implement sending files")
 
-    def download_file(self, file_id, file_name_raw):
-        """ Downloads a file """
-        logger.error("This backend does not support downloading files")
+    def download_file(self, file_id, file_name):
+        """Downloads a file using urllib.
+        Understands file_id as the url
+        """
+        return urllib.request.urlretrieve(file_id, file_name)

src/pybliotecario/backend/facebook_util.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+    Facebook backend
+
+    Using this backend will stat a flask server in the selected port.
+    Testing this backend is a bit of a pain as one has to be in a server
+    which facebook should be able to access with a valid SSL certificate.
+
+    For quick testing, I am using hthe following setup:
+    ~$ iptables -A INPUT -p tcp --dport 3000  -j ACCEPT
+    ~$ ngrok http <my_personal_server>:3000
+
+    And then I open the flask server in the port 3000 and give facebook
+    the ngrok url.
+    For actual deployment one would want to set up some actual server.
+"""
+
+import json
+import pathlib
+import logging
+import requests
+from pybliotecario.backend.basic_backend import Message, Backend
+
+_HAS_FLASK = True
+try:
+    from flask import Flask, request
+except ModuleNotFoundError:
+    _HAS_FLASK = False
+
+
+logger = logging.getLogger(__name__)
+
+FB_API = "https://graph.facebook.com/v2.12/me/messages"
+
+
+class FacebookMessage(Message):
+    """ Facebook implementation of the Message class """
+
+    _type = "facebook"
+    _group_info = None
+
+    def _parse_update(self, update):
+        """Receives an update in the form of a dictionary (that came from a json)
+        and fills in the _message_dict dictionary
+        """
+        # Check whether it is the correct object, otherwise out
+        if update.get("object") != "page":
+            logger.warning("Message not a object: page, ignoring")
+            logger.warning(update)
+            self.ignore = True
+            return
+        print(update)
+        msg_info = update["entry"][0]["messaging"][0]
+        # Check who sent it
+        sender_id = msg_info["sender"]["id"]
+        self._message_dict["chat_id"] = sender_id
+        # Get the msg
+        msg = msg_info["message"]
+        # get the text and parse it if necessary
+        text = msg.get("text")
+        self._message_dict["text"] = text
+        if text and text.startswith("/"):
+            self._parse_command(text)
+        # In facebook we have either text or image
+        # TODO: in facebook you can pass more than one img at once...
+        attachment = msg.get("attachments")
+        if attachment is not None:
+            at_info = attachment[0]
+            # Checked for images and files and seems to work
+            url = at_info["payload"]["url"]
+            self._message_dict["file_id"] = url
+            self._message_dict["text"] = url.split("?")[0].split("/")[-1]
+
+
+class FacebookUtil(Backend):
+    """This class handles all comunications with
+    Telegram"""
+
+    _message_class = FacebookMessage
+
+    def __init__(self, PAGE_TOKEN, VERIFY_TOKEN, host="0.0.0.0", port=3000, debug=False):
+        if not _HAS_FLASK:
+            # Do the error now
+            raise ModuleNotFoundError("No module named 'flask'")
+
+        self.page_access_token = PAGE_TOKEN
+        self.verify_token = VERIFY_TOKEN
+        self.port = port
+        self.host = host
+        app = Flask(__name__)
+        # Load the listener into the webhook endpoint
+        app.add_url_rule("/webhook", "webhook", self.listener, methods=["POST", "GET"])
+        self.flask_app = app
+        self.debug = debug
+        self.action_function = None
+        self.auth = {"access_token": self.page_access_token}
+
+    def validate_hook(self):
+        """Facebook needs to validate the webhook
+        This is a small utility to do so
+        """
+
+    def listener(self):
+        """ Main function flask will use to listen at the webhook endpoint """
+        if request.method == "GET":
+            if request.args.get("hub.verify_token") == self.verify_token:
+                return request.args.get("hub.challenge")
+            else:
+                return "incorrect"
+
+        if request.method == "POST":
+            msg = self._message_class(request.json)
+            self.action_function(msg)
+            logger.info(msg)
+            # After we have finished return a 200 Ok
+            return "All ok"
+
+    def act_on_updates(self, action_function, not_empty=False):
+        """Sets the action function to be used by the listener and then
+        opens the webhook to wait ofr updates and act on them
+        """
+        self.action_function = action_function
+        self.flask_app.run(host=self.host, port=self.port, debug=self.debug)
+
+    def _get_updates(self, not_empty=False):
+        """ This class skips get_updates and uses act_on_updates directly """
+        pass
+
+    def send_message(self, text, chat):
+        """ Sends a message response to facebook """
+        payload = {"message": {"text": text}, "recipient": {"id": chat}}
+        response = requests.post(FB_API, params=self.auth, json=payload)
+        return response.json()
+
+    def send_data(self, payload):
+        """Sends data to facebook messenger.
+        This method uses MultipartEncoder: https://toolbelt.readthedocs.io/
+        to stream multipart form-data
+        """
+        try:
+            from requests_toolbelt import MultipartEncoder
+        except ModuleNotFoundError as e:
+            raise ModuleNotFoundError(
+                "Install 'requests-toolbelt' to send images and files to facebook"
+            ) from e
+
+        encoded_payload = MultipartEncoder(payload)
+        header = {"Content-Type": encoded_payload.content_type}
+        response = requests.post(FB_API, params=self.auth, data=encoded_payload, headers=header)
+        return response.json()
+
+    def send_image(self, img_path, chat):
+        """Sends an image to facebook
+        Basically the requests form of the curl command here:
+        https://developers.facebook.com/docs/messenger-platform/send-messages#url
+        """
+        img = pathlib.Path(img_path)
+        payload = {
+            "recipient": json.dumps({"id": chat}),
+            "message": json.dumps(
+                {
+                    "attachment": {
+                        "type": "image",
+                        "payload": {"is_reusable": True},
+                    }
+                }
+            ),
+            "filedata": (img.stem, img.read_bytes(), f"image/{img.suffix[1:]}"),
+        }
+        return self.send_data(payload)
+
+    def send_file(self, filepath, chat):
+        """ Sends a file to fb, similar to send_image """
+        fff = pathlib.Path(filepath)
+        payload = {
+            "recipient": json.dumps({"id": chat}),
+            "message": json.dumps(
+                {
+                    "attachment": {
+                        "type": "file",
+                        "payload": {"is_reusable": True},
+                    }
+                }
+            ),
+            "filedata": (fff.name, fff.read_bytes()),
+        }
+        return self.send_data(payload)
+
+
+if __name__ == "__main__":
+    logger.info("Testing FB Util")
+    verify = "your_verify_token"
+    app_token = "your_app_key"
+    fb_util = FacebookUtil(app_token, verify, debug=True)
+    fb_util.act_on_updates(lambda x: print(x))

src/pybliotecario/backend/readme.md

@@ -0,0 +1,19 @@
+# Pybliotecario Backends
+
+There are some subtleties involved when implementing new backends.
+The main thing to take into account is that a new backend will require new API keys
+and, often, new ways to communicate with the remote server.
+
+For Telegram the process is quite simple and this program will guide you through the process
+of contacting the [botfather](https://t.me/botfather) and getting an API key.
+For Facebook instead the process is mre invovled.
+
+## Facebook backend
+
+To start the process one has two create an app in the [facebook's developer page](https://developers.facebook.com/)
+it will have to be associated with a Facebook page (one would talk to the bot through the page messenger).
+
+Once both page and app are created, you will want to add the `messenger` product to it and ensure that in your app
+subscription both `messaging` and `messaging_postbacks` are set.
+
+TODO: do a more detailed guide with screenshots and everything

src/pybliotecario/backend/telegram_util.py

@@ -87,20 +87,9 @@ def _parse_update(self, update):
             self._group_info = chat_data
 
         # Finally check whether the message looks like a command
-        if text and text.startswith("/"):
-            separate_command = text.split(" ", 1)
-            # Remove the / from the command
-            command = separate_command[0][1:]
-            # Absorb the @ in case it is a directed command!
-            if "@" in command:
-                command = command.split("@")[0]
-            # Check whether the command comes alone or has arguments
-            if len(separate_command) == 1:
-                text = ""
-            else:
-                text = separate_command[1]
-            self._message_dict["command"] = command
         self._message_dict["text"] = text
+        if text and text.startswith("/"):
+            self._parse_command(text)
 
     @property
     def is_group(self):
@@ -231,17 +220,16 @@ def send_file_by_url(self, file_url, chat):
         blabla = requests.post(self.send_doc, data=data)
         logger.info(blabla.status_code, blabla.reason, blabla.content)
 
-    def download_file(self, file_id, file_name_raw):
+    def download_file(self, file_id, file_name):
         """Download file defined by file_id
         to given file_name"""
         file_url = self._get_filepath(file_id)
         if not file_url:
             return None
-        file_name = file_name_raw
         n = 0
         while os.path.isfile(file_name):
-            filedir = os.path.dirname(file_name_raw)
-            basename = os.path.basename(file_name_raw)
+            filedir = os.path.dirname(file_name)
+            basename = os.path.basename(file_name)
             file_name = "{0}/n{1}-{2}".format(filedir, n, basename)
             n += 1
         return urllib.request.urlretrieve(file_url, file_name)

src/pybliotecario/pybliotecario.py

@@ -8,7 +8,7 @@
 import sys
 import os
 
-from pybliotecario.backend import TelegramUtil, TestUtil
+from pybliotecario.backend import TelegramUtil, TestUtil, FacebookUtil
 from pybliotecario.core_loop import main_loop
 
 # Modify argument_parser.py to read new arguments
@@ -103,6 +103,18 @@ def main(cmdline_arg=None, tele_api=None, config=None):
             tele_api = TelegramUtil(api_token, debug=args.debug)
         elif args.backend.lower() == "test":
             tele_api = TestUtil("/tmp/test_file.txt")
+        elif args.backend.lower() == "facebook":
+            try:
+                fb_config = config["FACEBOOK"]
+            except KeyError:
+                raise ValueError("No facebook section found for facebook in pybliotecario.ini")
+            verify_token = fb_config.get("verify")
+            app_token = fb_config.get("app_token")
+            tele_api = FacebookUtil(app_token, verify_token, debug=args.debug)
+            # Check whether we have chat id
+            chat_id = fb_config.get("chat_id")
+            if chat_id is not None:
+                config.set("DEFAULT", "chat_id", chat_id)
 
     on_cmdline.run_command(args, tele_api, config)