diff --git a/setup.py b/setup.py index 74a900a..8ec0204 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ import sys pybliotecario_name = "pybliotecario" -print(sys.argv) if "--with_name" in sys.argv: import socket @@ -15,8 +14,6 @@ # Remove it from the list of argument, nobody should know sys.argv.remove("--with_name") -print(sys.argv) - # Readup the readme README = (pathlib.Path(__file__).parent / "readme.md").read_text() setup( @@ -33,13 +30,13 @@ packages=find_packages("src"), install_requires=[ "numpy", + "requests", "regex", "arxiv", "pyowm", "psutil", "wikipedia", ], - extra_requires={"facebook": ["flask"]}, entry_points={ "console_scripts": [ "{0} = pybliotecario.pybliotecario:main".format(pybliotecario_name), diff --git a/src/pybliotecario/Message.py b/src/pybliotecario/Message.py deleted file mode 100644 index d54cac4..0000000 --- a/src/pybliotecario/Message.py +++ /dev/null @@ -1,116 +0,0 @@ -###### Esta clase tiene que ser redefinida de forma uqe definamos un MessageParser con una serie de opciones y un Message -###### donde el message parser es algo en plan que define una serie de opciones y el message pues tiene todos los getters - -import json - -registeredCommands = [] -import logging - -logger = logging.getLogger(__name__) - - -class Message: - # Variables that we are going to parse from json: - # chat_id - Id of the chat the message came from - # username - user who sent the message - # is_command - t/f - # isRegisteredCommand - t/f - # has_arguments - t/f - # is_group - t/f - # isFile - t/f - # fileId - file id - # text - actual content of the message (minus /command) - # command - command given in /command (or "") - # ignore - t/f (whether this message should be ignored) - - def __init__(self, jsonDict): - # ignore keys: - ign_keys = ["new_chat_participant", "left_chat_participant", "sticker", "game", "contact"] - msg = "message" - self.json = jsonDict - keys = jsonDict.keys() - if msg not in keys: - if "edited_message" in keys: - msg = "edited_message" - elif "edited_channel_post" in keys: - msg = "edited_channel_post" - try: - message = jsonDict[msg] - except: - logger.info(" >>>>> ") - logger.info(jsonDict) - raise Exception("Not a message or an edited message?") - msgKeys = message.keys() - if set(ign_keys) & set(msgKeys): - self.ignore = True - return - else: - self.ignore = False - self.has_arguments = False - chatData = message["chat"] - if "from" in message.keys(): - fromData = message["from"] - else: - fromData = chatData # something has changed or was this a special type of msg??? - # Populate general fields - # Check whetehr username exists, otherwise use name, otherwise, unknown - if "username" in fromData: - self.username = fromData["username"] - elif "first_name" in fromData: - self.username = fromData["first_name"] - elif "last_name" in fromData: - self.username = fromData["last_name"] - else: - self.username = "unknown_user" - self.chat_id = chatData["id"] - - # Check the filetyp of what we just received - if "photo" in msgKeys: - self.isFile = True - photoData = message["photo"][-1] - self.fileId = photoData["file_id"] - if "caption" in msgKeys: - self.text = message["caption"] - else: - self.text = "untitled" - if not self.text.endswith((".jpg", ".JPG", ".png", ".PNG")): - self.text += ".jpg" - elif "document" in msgKeys: - self.isFile = True - fileData = message["document"] - self.fileId = fileData["file_id"] - self.text = fileData["file_name"] - elif "sticker" in msgKeys: - self.isSticker = True - stickerData = message["sticker"] - self.stickerSet = stickerData["set_name"] - else: - self.text = message.get("text", "") - self.isFile = False - - # Check whether the msg comes from a group - self.is_group = chatData["type"] == "group" - - #  Now check whether the msg has the structure of a command - if self.text and self.text[0] == "/": - self.is_command = True - else: - self.is_command = False - self.command = "" - self.isRegisteredCommand = False - - if self.is_command: - all_text = self.text.split(" ", 1) - # Check whether the command comes by itself or has arguments - if len(all_text) > 1: - self.has_arguments = True - # Remove the / - self.command = all_text[0][1:] - # Absorb the @ in case is it a directed command - if "@" in self.command: - self.command = self.command.split("@")[0] - self.text = all_text[-1] - self.isRegisteredCommand = self.command in registeredCommands - - def __str__(self): - return json.dumps(self.json) diff --git a/src/pybliotecario/__init__.py b/src/pybliotecario/__init__.py index 6e9cc54..8c0d5d5 100644 --- a/src/pybliotecario/__init__.py +++ b/src/pybliotecario/__init__.py @@ -1,3 +1 @@ __version__ = "2.0.0" - -from pybliotecario.Message import Message diff --git a/src/pybliotecario/argument_parser.py b/src/pybliotecario/argument_parser.py index 9b0bf05..b6aa7a5 100644 --- a/src/pybliotecario/argument_parser.py +++ b/src/pybliotecario/argument_parser.py @@ -78,7 +78,7 @@ def configure_telegram(main_folder): teleAPI = TelegramUtil(token, timeout=20) while True: all_updates = teleAPI.get_updates(not_empty=True) - from pybliotecario.Message import Message + from pybliotecario.backend.telegram_util import TelegramMessage update = Message(all_updates[0]) print("Message received: {0}".format(update.text)) @@ -163,7 +163,9 @@ def parse_args(args): """ Wrapper for ArgumentParser """ parser = ArgumentParser() parser.add_argument( - "--init", help="Wizard to configure the pybliotecario for the first time", action=InitAction + "--init", + help="Wizard to configure the pybliotecario for the first time", + action=InitAction, ) parser.add_argument("--config_file", help="Define a custom configuration file") parser.add_argument("--backend", help="Choose backend", type=str, default="Telegram") diff --git a/src/pybliotecario/backend/backend_test.py b/src/pybliotecario/backend/backend_test.py index f224a76..ef08833 100644 --- a/src/pybliotecario/backend/backend_test.py +++ b/src/pybliotecario/backend/backend_test.py @@ -3,18 +3,19 @@ without communication with any service """ -import copy import pathlib -import numpy as np from datetime import datetime +import numpy as np -from pybliotecario.Message import Message +from pybliotecario.backend.telegram_util import TelegramMessage TESTID = 1234 # chat id for the test backend _TESTUSER = "hiro" -class TestMessage(Message): +class TestMessage(TelegramMessage): + """ Copy of the TelegramMessage class """ + _type = "Test" @@ -122,9 +123,3 @@ def is_msg_in_file(self, msg): This is something that is only useful for the TestUtil backend""" read_text = self.comm_file.read_text() return msg in read_text - - -if __name__ == "__main__": - cls = TestUtil("/tmp/test.txt") - res = cls._get_updates() - msgs = [TestMessage(i) for i in res] diff --git a/src/pybliotecario/backend/basic_backend.py b/src/pybliotecario/backend/basic_backend.py new file mode 100644 index 0000000..63a4ad4 --- /dev/null +++ b/src/pybliotecario/backend/basic_backend.py @@ -0,0 +1,153 @@ +""" + Base/abstract backend classes: + + - Message + - Backend + + Each backend should implement its own message type and inherit from backend +""" + +from abc import ABC, abstractmethod, abstractproperty +import logging +import json + +logger = logging.getLogger(__name__) + + +class Message(ABC): + """ + Base message class + + Any implementation of the message should either save a dictionary + ``_message_dict`` with the following attributes (through _parse_update) + or implement its own way of getting the different values + """ + + _type = "Abstract" + _original = None + + _message_dict = { + "chat_id": None, + "username": None, + "command": None, + "file_id": None, + "text": None, + "ignore": False, + } + + def __init__(self, update): + self._original = update + self._parse_update(update) + # After the information is parsed, log the message! + logger.info("New message: %s", self) + + def __str__(self): + return json.dumps(self._message_dict) + + @abstractmethod + def _parse_update(self, update): + """ Parse the update and fill in _message_dict """ + return None + + @property + def chat_id(self): + """ Returns the chat id """ + return self._message_dict["chat_id"] + + @property + def username(self): + """ Returns the username """ + return self._message_dict["username"] + + @property + def text(self): + """ Returns the content of the message """ + return self._message_dict.get("text") + + @property + def is_command(self): + """ Returns true if the message is a command """ + return self._message_dict.get("command") is not None + + @property + def command(self): + """ Returns the command contained in the message """ + return self._message_dict.get("command") + + @property + def is_file(self): + """ Returns true if the message is a file """ + return self._message_dict.get("file_id") is not None + + @property + def file_id(self): + """ Returns the id of the file """ + return self._message_dict.get("file_id") + + @property + def has_arguments(self): + """ Returns true if the message is a command with arguments """ + return self.is_command and self.text is not None + + @property + def ignore(self): + """ Returns true if the message is to be ignored """ + return self._message_dict.get("ignore", False) + + @ignore.setter + def ignore(self, val): + self._message_dict["ignore"] = val + + +class Backend(ABC): + """ + Main backend class for inheritting. + + It provides a number of base functions and wrappers + + The minimum set of methods and properties the backend must define are: + + _message_class: a reference to the Message class of the backend + _get_updates: a method that should return a list of updates to act upon + send_message: a method to communicate messages + + Others: + - send_image + - send_file + - download_file + + """ + + @abstractmethod + def _get_updates(self, not_empty=False): + """ Retrieve updates """ + + @abstractmethod + def send_message(self, text, chat): + """ Sends a message to the chat """ + + @abstractproperty + def _message_class(self): + pass + + def act_on_updates(self, action_function, not_empty=False): + """ + Receive the input using _get_updates, parse it with + the telegram message class and act in consequence + """ + all_updates = self._get_updates(not_empty=not_empty) + for update in all_updates: + msg = self._message_class(update) + action_function(msg) + + def send_image(self, img_path, chat): + """ Sends an image """ + logger.error("This backend does not implement sending files") + + def send_file(self, filepath, chat): + """ Sends a file """ + logger.error("This backend does not implement sending files") + + def download_file(self, file_id, file_name_raw): + """ Downloads a file """ + logger.error("This backend does not support downloading files") diff --git a/src/pybliotecario/backend/telegram_util.py b/src/pybliotecario/backend/telegram_util.py index 2560b8f..9b56097 100644 --- a/src/pybliotecario/backend/telegram_util.py +++ b/src/pybliotecario/backend/telegram_util.py @@ -1,29 +1,119 @@ +""" + Telegram backend +""" + #!/usr/bin/env python3 import json import os.path import urllib +import logging import requests -from pybliotecario import Message +from pybliotecario.backend.basic_backend import Message, Backend TELEGRAM_URL = "https://api.telegram.org/" -import logging logger = logging.getLogger(__name__) +# Keys included in telegram chats that basically are telling you to ignore it +IGNOREKEYS = set(["new_chat_participant", "left_chat_participant", "sticker", "game", "contact"]) + def log_request(status_code, reason, content): """ Log the status of the send requests """ result = "Request sent, status code: {0} - {1}: {2}".format(status_code, reason, content) logger.info(result) + class TelegramMessage(Message): + """ Telegram implementation of the Message class """ + _type = "Telegram" + _group_info = None + + def _parse_update(self, update): + """Receives an update in the form of a dictionary (that came from a json) + and fills in the _message_dict dictionary + """ + keys = update.keys() + # First check whether this is a message, edited message or a channel post + msg_types = ["message", "edited_message", "edited_channel_post"] + msg = None + for msg_type in msg_types: + if msg_type in keys: + msg = msg_type + if msg is None: + logger.warning(f"Message not in {msg_types}, ignoring") + logger.warning(update) + self.ignore = True + return + message = update[msg] + # Get the keys of the message (and check whether it should be ignored) + msg_keys = message.keys() + if set(msg_keys) & IGNOREKEYS: + self.ignore = True + return + # Now get the chat data and id + chat_data = message["chat"] + self._message_dict["chat_id"] = chat_data["id"] + # TODO test this part of the parser as this 'from' was a legacy thing at some point + from_data = message.get("from", chat_data) + # Populate the user (in the list, last has more priority) + username = "unknown_user" + for user_naming in ["last_name", "first_name", "username"]: + username = from_data.get(user_naming, username) + self._message_dict["username"] = username -class TelegramUtil: + # Check the filetype + text = None + if "photo" in message: + # If it is a photo, get the file id and use the caption as the title + photo_data = message["photo"][-1] + self._message_dict["file_id"] = photo_data["file_id"] + text = message.get("caption", "untitled") + if not text.endswith((".jpg", ".JPG", ".png", ".PNG")): + text += ".jpg" + elif "document" in message: + # If it is a document, teleram gives you everything you need + file_dict = message["document"] + self._message_dict["file_id"] = file_dict["file_id"] + text = file_dict["file_name"] + else: + # Normal text message + text = message.get("text", "") + + # In Telegram we can also have groups + if "group" in chat_data: + self._group_info = chat_data + + # Finally check whether the message looks like a command + if text and text.startswith("/"): + separate_command = text.split(" ", 1) + # Remove the / from the command + command = separate_command[0][1:] + # Absorb the @ in case it is a directed command! + if "@" in command: + command = command.split("@")[0] + # Check whether the command comes alone or has arguments + if len(separate_command) == 1: + text = "" + else: + text = separate_command[1] + self._message_dict["command"] = command + self._message_dict["text"] = text + + @property + def is_group(self): + """ Returns true if the message was from a group """ + return self._group_info is not None + + +class TelegramUtil(Backend): """This class handles all comunications with Telegram""" + _message_class = TelegramMessage + def __init__(self, TOKEN, debug=False, timeout=300): self.offset = None self.debug = debug @@ -64,18 +154,18 @@ def __re_offset(self, updates): li.append(int(update["update_id"])) self.offset = max(li) + 1 - def _get_filepath(self, fileId): + def _get_filepath(self, file_id): """Given a file id, retrieve the URI of the file in the remote server """ - url = self.get_file + "?file_id={0}".format(fileId) - json = self.__get_json_from_url(url) + url = self.get_file + "?file_id={0}".format(file_id) + jsonret = self.__get_json_from_url(url) # was it ok? - if json["ok"]: - fpath = json["result"]["file_path"] + if jsonret["ok"]: + fpath = jsonret["result"]["file_path"] return self.base_fileURL + fpath else: - logger.info(json["error_code"]) + logger.info(jsonret["error_code"]) logger.info("Here's all the information we have on this request") logger.info("This is the url we have used") logger.info(url) @@ -110,16 +200,6 @@ def _get_updates(self, not_empty=False): self.__re_offset(result) return result - def act_on_updates(self, action_function, not_empty=False): - """ - Receive the input using _get_updates, parse it with - the telegram message class and act in consequence - """ - all_updates = self._get_updates(not_empty=not_empty) - for update in all_updates: - msg = TelegramMessage(update) - action_function(msg) - def send_message(self, text, chat): """ Send a message to a given chat """ text = urllib.parse.quote_plus(text) @@ -151,10 +231,10 @@ def send_file_by_url(self, file_url, chat): blabla = requests.post(self.send_doc, data=data) logger.info(blabla.status_code, blabla.reason, blabla.content) - def download_file(self, fileId, file_name_raw): - """Download file defined by fileId + def download_file(self, file_id, file_name_raw): + """Download file defined by file_id to given file_name""" - file_url = self._get_filepath(fileId) + file_url = self._get_filepath(file_id) if not file_url: return None file_name = file_name_raw @@ -169,16 +249,16 @@ def download_file(self, fileId, file_name_raw): if __name__ == "__main__": logger.info("Testing TelegramUtil") - TOKEN = "must put a token here to test" - ut = TelegramUtil(TOKEN, debug=True) + token = "must put a token here to test" + ut = TelegramUtil(token, debug=True) results = ut._get_updates() - for result in results: + for res in results: logger.info("Complete json:") - logger.info(result) - message = result["message"] - chat_id = message["chat"]["id"] - txt = message["text"] - logger.info("Message from {0}: {1}".format(chat_id, txt)) + logger.info(res) + msg_example = res["message"] + chat_id = msg_example["chat"]["id"] + txt = msg_example["text"] + logger.info("Message from %s: %s", chat_id, txt) ut.send_message("Message received", chat_id) ut.timeout = 1 ut._get_updates() # Use the offset to confirm updates diff --git a/src/pybliotecario/core_loop.py b/src/pybliotecario/core_loop.py index baa9cef..d2a7ec3 100644 --- a/src/pybliotecario/core_loop.py +++ b/src/pybliotecario/core_loop.py @@ -1,13 +1,12 @@ from datetime import datetime import os -from pybliotecario.Message import Message import pybliotecario.on_cmd_message as on_cmd_message import logging logger = logging.getLogger(__name__) # After which number of continuous exceptions do we actually fail -_FAILTHRESHOLD = 100 +_FAILTHRESHOLD = 20 def monthly_folder(base_main_folder): @@ -68,15 +67,9 @@ def main_loop(tele_api, config=None, clear=False): instance of the message as the argument. This allows the tele_api to do things asynchronously if needed be """ - # Check whether we have an accepted user - if config: - accepted_user = config["DEFAULT"].get("chat_id") - else: - accepted_user = None main_folder = config["DEFAULT"]["main_folder"] except_counter = 0 - # Generate the function to act on Messages def act_on_message(message): """This function receives a pybliotecario.Message and @@ -89,22 +82,12 @@ def act_on_message(message): chat_id = message.chat_id if message.is_command: # Call the selected command and act on the message - response = on_cmd_message.act_on_telegram_command(tele_api, message, config) - # If response can be sent to the chat, do so - # TODO: make this part of a greater BACKEND class - if isinstance(response, str): - tele_api.send_message(response, chat_id) - elif isinstance(response, dict): - if response.get("isfile"): - filepath = response.get("filename") - tele_api.send_file(filepath, chat_id) - if response.get("delete"): - os.remove(filepath) - elif message.isFile: + on_cmd_message.act_on_telegram_command(tele_api, message, config) + elif message.is_file: # If the message is a file, save the file and we are done file_name = message.text.replace(" ", "") file_path = "{0}/{1}".format(monthly_folder(main_folder), file_name) - result = tele_api.download_file(message.fileId, file_path) + result = tele_api.download_file(message.file_id, file_path) if result: tele_api.send_message("¡Archivo recibido y guardado!", chat_id) logger.info("File saved to %s", file_path) @@ -117,9 +100,11 @@ def act_on_message(message): write_to_daily_log(main_folder, message.text) random_msg = still_alive() tele_api.send_message(random_msg, chat_id) + except_counter = 0 except Exception as e: logger.error(f"This message produced an exception: {e}") if clear and except_counter < _FAILTHRESHOLD: + except_counter += 1 # Ignore exceptions until we reach the threshold logger.info(message) logger.info("Going for the next message") @@ -127,67 +112,3 @@ def act_on_message(message): raise e tele_api.act_on_updates(act_on_message, not_empty=True) - - -# -# def main_loop(tele_api, config=None, clear=False): -# """ -# This function activates a "listener" and waits for updates from Telegram -# No matter what the update is about, we first store the content and then -# if it is a command, we act on the command -# """ -# if config: -# accepted_user = config["DEFAULT"]["chat_id"] -# else: -# accepted_user = None -# main_folder = config["DEFAULT"]["main_folder"] -# # Get updates from Telegram -# raw_updates = tele_api.get_updates(not_empty=True) -# updates = [Message(update) for update in raw_updates] -# # Act on those updates -# for update in updates: -# logger.info(update.json) -# try: -# if update.ignore: -# continue -# chat_id = update.chat_id -# if update.is_command: -# # Calls select command and act on the message -# # the function will receive the whole telegram API so it is allowed to send msgs directly -# # it can choose to send back a response instead -# response = on_cmd_message.act_on_telegram_command(tele_api, update, config) -# # if response is text, or file, it will be sent to the chat -# if isinstance(response, str): -# tele_api.send_message(response, chat_id) -# elif isinstance(response, dict): -# if response["isfile"]: -# filepath = response["filename"] -# tele_api.send_file(filepath, chat_id) -# if response["delete"]: -# os.remove(filepath) -# elif update.isFile: -# # If the update is a file, save the file and we are done -# file_name = update.text.replace(" ", "") -# file_path = "{0}/{1}".format(monthly_folder(main_folder), file_name) -# result = tele_api.download_file(update.fileId, file_path) -# if result: -# tele_api.send_message("¡Archivo recibido y guardado!", chat_id) -# logger.info("File saved to {0}".format(file_path)) -# else: -# tele_api.send_message("There was some problem with this, sorry", chat_id) -# logger.info( -# "Since there was some problem, let's open a pdb console here and you decide what to do" -# ) -# -# else: -# # Otherwise just save the msg to the log and send a funny reply -# write_to_daily_log(main_folder, update.text) -# random_msg = still_alive() -# tele_api.send_message(random_msg, chat_id) -# except Exception as e: -# # If we are in clear mode, we want to recapture updates to ensure we clear the ones that produce a fail -# # in principle in clear mode we don't care what the fail is about, we just want to clear the failure -# if clear: -# logger.info("\n > > This update produced an exception: {0}\n\n".format(e)) -# else: -# raise e diff --git a/src/pybliotecario/pybliotecario.py b/src/pybliotecario/pybliotecario.py index 05bad6e..b5e2c2c 100755 --- a/src/pybliotecario/pybliotecario.py +++ b/src/pybliotecario/pybliotecario.py @@ -81,7 +81,8 @@ def main(cmdline_arg=None, tele_api=None, config=None): main_folder = defaults.get("main_folder") if not main_folder: logger.warning( - "No 'default:main_folder' option set in %s, using /tmp/", args.config_file + "No 'default:main_folder' option set in %s, using /tmp/", + args.config_file, ) main_folder = "/tmp/" logger_setup(main_folder + "/info.log", debug=args.debug) @@ -94,7 +95,8 @@ def main(cmdline_arg=None, tele_api=None, config=None): api_token = defaults.get("token") if not api_token: logger.error( - "No 'default:token' option set in %s, run --init option", args.config_file + "No 'default:token' option set in %s, run --init option", + args.config_file, ) sys.exit(-1)