Various Enhancements and Fixes (#61)

* added purgeme

* Bug Fixes and Upstreams

- added missing stuff
- fixed purgeme

* Update gdrive.py

* Update gizoogle.py

* Update aiohttp_helper.py

* Update config.py

Aptfile

@@ -1,5 +1,4 @@
 tree
-wget
 wget2
 pv
 p7zip-full

app.json

@@ -12,14 +12,14 @@
     ],
     "repository": "https://github.com/code-rgb/USERGE-X",
     "website": "https://github.com/code-rgb/USERGE-X",
-    "success_url": "https://t.me/x_xtests",
+    "success_url": "https://t.me/userge_x",
     "env": {
         "API_ID": {"description": "Get this value from https://my.telegram.org"},
         "API_HASH": {"description": "Get this value from https://my.telegram.org"},
         "DATABASE_URL": {"description": "Mongodb url from https://cloud.mongodb.com/"},
         "HEROKU_APP_NAME": {"description": "Given app name to the Heroku App"},
         "LOG_CHANNEL_ID": {"description": "Telegram Log Channel ID, Note: Also add your Bot to LOG CHANNEL !"},
-        "HEROKU_API_KEY": {"description": "Get a Heroku API key from http://dashboard.heroku.com/account",
+        "HEROKU_API_KEY": {"description": "[Required] Get a Heroku API key from http://dashboard.heroku.com/account",
                            "required": false},
         "HU_STRING_SESSION": {
             "description": "Get this using 'https://GenUserGeString.usergeuserbot.repl.run' or `bash genStr`",

init/utils.sh

@@ -83,6 +83,10 @@ fetchBranches() {
     done
 }
 
+updateBuffer() {
+    git config http.postBuffer 524288000
+}
+
 upgradePip() {
     pip3 install -U pip &> /dev/null
 }
@@ -113,7 +117,7 @@ printLogo() {
                  X::::::X     X::::::X                
                  X:::::X       X:::::X                
                  X:::::X       X:::::X                
-                 XXXXXXX       XXXXXXX                                                         
+                 XXXXXXX       XXXXXXX                
 '
     printLine
 }

userge/config.py

@@ -47,7 +47,6 @@ class Config:
     INSTA_PASS = os.environ.get("INSTA_PASS")
     UPSTREAM_REPO = os.environ.get("UPSTREAM_REPO")
     UPSTREAM_REMOTE = os.environ.get("UPSTREAM_REMOTE")
-    SCREENSHOT_API = os.environ.get("SCREENSHOT_API", None)
     SPAM_WATCH_API = os.environ.get("SPAM_WATCH_API", None)
     CURRENCY_API = os.environ.get("CURRENCY_API", None)
     OCR_SPACE_API_KEY = os.environ.get("OCR_SPACE_API_KEY", None)
@@ -94,11 +93,14 @@ class Config:
     IMGFLIP_ID = os.environ.get('IMGFLIP_ID', None)
     IMGFLIP_PASS = os.environ.get('IMGFLIP_PASS', None)
     ALLOW_NSFW = os.environ.get("ALLOW_NSFW", "False")
-    pmlog_grp = os.environ.get("PM_LOG_GROUP_ID")
+    pmlog_grp = os.environ.get("PM_LOG_GROUP_ID", None)
     PM_LOG_GROUP_ID = int(pmlog_grp) if pmlog_grp else None
     PM_LOGGING = False
     DEEP_AI = os.environ.get("DEEP_AI", None)
-
+    ### Last FM
+    LASTFM_USERNAME = os.environ.get("LASTFM_USERNAME", None)
+    LASTFM_API_KEY = os.environ.get("LASTFM_API_KEY", None)
+
 
 def get_version() -> str:
     """ get userge version """

userge/core/client.py

@@ -34,13 +34,6 @@
 _START_TIME = time.time()
 
 
-def _shutdown() -> None:
-    _LOG.info(_LOG_STR, 'received stop signal, cancelling tasks ...')
-    for task in asyncio.all_tasks():
-        task.cancel()
-    _LOG.info(_LOG_STR, 'all tasks cancelled !')
-
-
 async def _complete_init_tasks() -> None:
     if not _INIT_TASKS:
         return
@@ -58,15 +51,15 @@ def is_bot(self) -> bool:
 
     @property
     def uptime(self) -> str:
-        """ returns userge uptime """
+        """ returns USERGE-X uptime """
         return time_formatter(time.time() - _START_TIME)
 
     async def finalize_load(self) -> None:
         """ finalize the plugins load """
         await asyncio.gather(_complete_init_tasks(), self.manager.init())
 
     async def load_plugin(self, name: str, reload_plugin: bool = False) -> None:
-        """ Load plugin to Userge """
+        """ Load plugin to USERGE-X """
         _LOG.debug(_LOG_STR, f"Importing {name}")
         _IMPORTED.append(
             importlib.import_module(f"userge.plugins.{name}"))
@@ -77,8 +70,7 @@ async def load_plugin(self, name: str, reload_plugin: bool = False) -> None:
         if hasattr(plg, '_init'):
             # pylint: disable=protected-access
             if asyncio.iscoroutinefunction(plg._init):
-                _INIT_TASKS.append(
-                    asyncio.get_event_loop().create_task(plg._init()))
+                _INIT_TASKS.append(self.loop.create_task(plg._init()))
         _LOG.debug(_LOG_STR, f"Imported {_IMPORTED[-1].__name__} Plugin Successfully")
 
     async def _load_plugins(self) -> None:
@@ -114,7 +106,7 @@ async def reload_plugins(self) -> int:
 class _UsergeBot(_AbstractUserge):
     """ UsergeBot, the bot """
     def __init__(self, **kwargs) -> None:
-        _LOG.info(_LOG_STR, "Setting UsergeBot Configs")
+        _LOG.info(_LOG_STR, "Setting X-bot Configs")
         super().__init__(session_name=":memory:", **kwargs)
 
     @property
@@ -124,12 +116,12 @@ def ubot(self) -> 'Userge':
 
 
 class Userge(_AbstractUserge):
-    """ Userge, the userbot """
+    """ USERGE-X, the userbot """
 
     has_bot = bool(Config.BOT_TOKEN)
 
     def __init__(self, **kwargs) -> None:
-        _LOG.info(_LOG_STR, "Setting Userge Configs")
+        _LOG.info(_LOG_STR, "Setting USERGE-X Configs")
         kwargs = {
             'api_id': Config.API_ID,
             'api_hash': Config.API_HASH,
@@ -145,7 +137,7 @@ def __init__(self, **kwargs) -> None:
 
     @property
     def bot(self) -> Union['_UsergeBot', 'Userge']:
-        """ returns usergebot """
+        """ returns X-bot """
         if self._bot is None:
             if Config.BOT_TOKEN:
                 return self
@@ -158,45 +150,58 @@ async def start(self) -> None:
         _LOG.info(_LOG_STR, "Starting USERGE-X")
         await super().start()
         if self._bot is not None:
-            _LOG.info(_LOG_STR, "Starting USERGE-X Bot")
+            _LOG.info(_LOG_STR, "Starting X-bot")
             await self._bot.start()
         await self._load_plugins()
 
     async def stop(self) -> None:  # pylint: disable=arguments-differ
         """ stop client and bot """
         if self._bot is not None:
-            _LOG.info(_LOG_STR, "Stopping USERGE-X Bot")
+            _LOG.info(_LOG_STR, "Stopping X-bot")
             await self._bot.stop()
-        _LOG.info(_LOG_STR, "Stopping USERGE-X")
+        _LOG.info(_LOG_STR, "Stopping Userge")
         await super().stop()
         await pool._stop()  # pylint: disable=protected-access
 
     def begin(self, coro: Optional[Awaitable[Any]] = None) -> None:
-        """ start userge """
-        loop = asyncio.get_event_loop()
-        loop.add_signal_handler(signal.SIGHUP, _shutdown)
-        loop.add_signal_handler(signal.SIGTERM, _shutdown)
-        run = loop.run_until_complete
+        """ start USERGE-X """
+        lock = asyncio.Lock()
+        running_tasks: List[asyncio.Task] = []
+
+        async def _finalize() -> None:
+            async with lock:
+                for task in running_tasks:
+                    task.cancel()
+                if self.is_initialized:
+                    await self.stop()
+                # pylint: disable=expression-not-assigned
+                [t.cancel() for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+                await self.loop.shutdown_asyncgens()
+                self.loop.stop()
+                _LOG.info(_LOG_STR, "Loop Stopped !")
+
+        async def _shutdown(sig: signal.Signals) -> None:
+            _LOG.info(_LOG_STR, f"Received Stop Signal [{sig.name}], Exiting USERGE-X ...")
+            await _finalize()
+
+        for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGINT):
+            self.loop.add_signal_handler(
+                sig, lambda sig=sig: self.loop.create_task(_shutdown(sig)))
+        self.loop.run_until_complete(self.start())
+        for task in self._tasks:
+            running_tasks.append(self.loop.create_task(task()))
+        logbot.edit_last_msg("USERGE-X has Started Successfully !")
+        logbot.end()
         try:
-            run(self.start())
-            running_tasks: List[asyncio.Task] = []
-            for task in self._tasks:
-                running_tasks.append(loop.create_task(task()))
             if coro:
                 _LOG.info(_LOG_STR, "Running Coroutine")
-                run(coro)
+                self.loop.run_until_complete(coro)
             else:
                 _LOG.info(_LOG_STR, "Idling USERGE-X")
-                logbot.edit_last_msg("USERGE-X has Started Successfully !")
-                logbot.end()
                 idle()
-            _LOG.info(_LOG_STR, "Exiting USERGE-X")
-            for task in running_tasks:
-                task.cancel()
-            run(self.stop())
-            run(loop.shutdown_asyncgens())
-        except asyncio.exceptions.CancelledError:
+            self.loop.run_until_complete(_finalize())
+        except (asyncio.exceptions.CancelledError, RuntimeError):
             pass
         finally:
-            if not loop.is_running():
-                loop.close()
+            self.loop.close()
+            _LOG.info(_LOG_STR, "Loop Closed !")

userge/core/methods/decorators/add_task.py

@@ -16,7 +16,7 @@
 
 
 class AddTask(RawDecorator):  # pylint: disable=missing-class-docstring
-    def add_task(self, func: Callable[[Any], Any]) -> Callable[[Any], Any]:
+    def add_task(self, func: Callable[[], Any]) -> Callable[[], Any]:
         """ add tasks """
         self._tasks.append(func)
         return func

userge/core/methods/decorators/raw_decorator.py

@@ -197,8 +197,10 @@ async def _both_have_perm(flt: Union['types.raw.Command', 'types.raw.Filter'],
     if flt.check_invite_perm and not (
             (user.can_all or user.can_invite_users) and bot.can_invite_users):
         return False
-    return bool(not flt.check_pin_perm or (
-            (user.can_all or user.can_pin_messages) and bot.can_pin_messages))
+    if flt.check_pin_perm and not (
+            (user.can_all or user.can_pin_messages) and bot.can_pin_messages):
+        return False
+    return True
 
 
 class RawDecorator(RawClient):
@@ -207,7 +209,7 @@ class RawDecorator(RawClient):
 
     def __init__(self, **kwargs) -> None:
         self.manager = types.new.Manager(self)
-        self._tasks: List[Callable[[Any], Any]] = []
+        self._tasks: List[Callable[[], Any]] = []
         super().__init__(**kwargs)
 
     def on_filters(self, filters: RawFilter, group: int = 0,

userge/core/methods/messages/edit_message_text.py

@@ -17,6 +17,7 @@
 from pyrogram.types import InlineKeyboardMarkup
 
 from userge import Config
+from userge.utils import secure_text
 from ...ext import RawClient
 from ... import types
 
@@ -82,6 +83,8 @@ async def edit_message_text(self,  # pylint: disable=arguments-differ
         Raises:
             RPCError: In case of a Telegram RPC error.
         """
+        if text and chat_id != Config.LOG_CHANNEL_ID:
+            text = secure_text(str(text))
         msg = await super().edit_message_text(chat_id=chat_id,
                                               message_id=message_id,
                                               text=text,
@@ -90,8 +93,7 @@ async def edit_message_text(self,  # pylint: disable=arguments-differ
                                               reply_markup=reply_markup)
         module = inspect.currentframe().f_back.f_globals['__name__']
         if log:
-            args = (msg, module) if isinstance(log, bool) else (msg, log)
-            await self._channel.fwd_msg(*args)
+            await self._channel.fwd_msg(msg, module if isinstance(log, bool) else log)
         del_in = del_in or Config.MSG_DELETE_TIMEOUT
         if del_in > 0:
             await asyncio.sleep(del_in)

userge/core/methods/messages/send_as_file.py

@@ -16,7 +16,8 @@
 
 import aiofiles
 
-from userge import logging
+from userge import logging, Config
+from userge.utils import secure_text
 from ...ext import RawClient
 from ... import types
 
@@ -69,17 +70,18 @@ async def send_as_file(self,
         Returns:
             On success, the sent Message is returned.
         """
+        if text and chat_id != Config.LOG_CHANNEL_ID:
+            text = secure_text(str(text))
         async with aiofiles.open(filename, "w+", encoding="utf8") as out_file:
             await out_file.write(text)
         _LOG.debug(_LOG_STR, f"Uploading {filename} To Telegram")
         msg = await self.send_document(chat_id=chat_id,
                                        document=filename,
-                                       caption=caption,
+                                       caption=caption[:1024],
                                        disable_notification=True,
                                        reply_to_message_id=reply_to_message_id)
         os.remove(filename)
         module = inspect.currentframe().f_back.f_globals['__name__']
         if log:
-            args = (msg, module) if isinstance(log, bool) else (msg, log)
-            await self._channel.fwd_msg(*args)
+            await self._channel.fwd_msg(msg, module if isinstance(log, bool) else log)
         return types.bound.Message.parse(self, msg, module=module)

userge/core/methods/messages/send_message.py

@@ -19,6 +19,7 @@
     ReplyKeyboardRemove, ForceReply)
 
 from userge import Config
+from userge.utils import secure_text
 from ...ext import RawClient
 from ... import types
 
@@ -91,6 +92,8 @@ async def send_message(self,  # pylint: disable=arguments-differ
         Returns:
             :obj:`Message`: On success, the sent text message or True is returned.
         """
+        if text and chat_id != Config.LOG_CHANNEL_ID:
+            text = secure_text(str(text))
         msg = await super().send_message(chat_id=chat_id,
                                          text=text,
                                          parse_mode=parse_mode,
@@ -101,8 +104,7 @@ async def send_message(self,  # pylint: disable=arguments-differ
                                          reply_markup=reply_markup)
         module = inspect.currentframe().f_back.f_globals['__name__']
         if log:
-            args = (msg, module) if isinstance(log, bool) else (msg, log)
-            await self._channel.fwd_msg(*args)
+            await self._channel.fwd_msg(msg, module if isinstance(log, bool) else log)
         del_in = del_in or Config.MSG_DELETE_TIMEOUT
         if del_in > 0:
             await asyncio.sleep(del_in)

userge/core/methods/utils/terminate.py

@@ -10,14 +10,21 @@
 
 __all__ = ['Terminate']
 
+import asyncio
+
 from ...ext import RawClient
 
 
 class Terminate(RawClient):  # pylint: disable=missing-class-docstring
     async def terminate(self) -> None:
         """ terminate userge """
         if not self.no_updates:
+            for _ in range(self.workers):
+                self.dispatcher.updates_queue.put_nowait(None)
             for task in self.dispatcher.handler_worker_tasks:
-                task.cancel()
+                try:
+                    await asyncio.wait_for(task, timeout=0.3)
+                except asyncio.TimeoutError:
+                    task.cancel()
             self.dispatcher.handler_worker_tasks.clear()
         await super().terminate()

userge/core/types/bound/message.py

@@ -379,7 +379,7 @@ async def edit(self,
                                        disable_web_page_preview=disable_web_page_preview,
                                        reply_markup=reply_markup)
                 if isinstance(msg, Message):
-                    self.message_id = msg.message_id
+                    self.message_id = msg.message_id  # pylint: disable=W0201
                 return msg
             raise m_er
 

userge/core/types/new/channel_logger.py

@@ -134,7 +134,7 @@ async def store(self,
                 pass pyrogram.Message object which want to forward.
 
             caption (`str`, *optional*):
-                Text or Cpation of the message to be sent.
+                Text or Caption of the message to be sent.
 
         Returns:
             message_id on success or None
@@ -201,10 +201,9 @@ async def forward_stored(self,
         if caption:
             u_dict = await client.get_user_dict(user_id)
             chat = await client.get_chat(chat_id)
-            u_dict.update(
-                {'chat': chat.title or "this group", 'count': chat.members_count}
-            )
-
+            u_dict.update({
+                'chat': chat.title if chat.title else "this group",
+                'count': chat.members_count})
             caption = caption.format_map(SafeDict(**u_dict))
         file_id, file_ref = get_file_id_and_ref(message)
         caption, buttons = parse_buttons(caption)