From fba036529ff66d6e82347f29c88794d9bc6c01bc Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 28 Feb 2022 23:17:32 -0800 Subject: [PATCH 01/78] add parse_core_permissions for more efficient and less restrictive permission parsing --- libs/lib_parsers.py | 65 ++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/libs/lib_parsers.py b/libs/lib_parsers.py index 238c9bb..f47e98c 100644 --- a/libs/lib_parsers.py +++ b/libs/lib_parsers.py @@ -28,7 +28,7 @@ from lib_encryption_wrapper import encrypted_reader import lib_constants as constants -from typing import Callable, Iterable, Optional, Any, Tuple, Dict, Union, List, TypeVar +from typing import Callable, Iterable, Optional, Any, Tuple, Dict, Union, List, TypeVar, Literal, overload import lib_lexdpyk_h as lexdpyk # Import re here to trick type checker into using re stubs even if importlib grabs re2, they (should) have the same stubs @@ -296,10 +296,51 @@ def _parse_role_perms(author: discord.Member, permrole: str) -> bool: Permtype = Union[str, Tuple[str, Callable[[discord.Message], bool]]] +@overload +def parse_core_permissions(channel: discord.TextChannel, member: discord.Member, mconf: Dict[str, str], perms: Literal["everyone"]) -> Literal[True]: + ... + + +@overload +def parse_core_permissions(channel: discord.TextChannel, member: discord.Member, mconf: Dict[str, str], perms: Literal["moderator", "administrator", "owner"]) -> bool: + ... + + +@overload +def parse_core_permissions(channel: discord.TextChannel, member: discord.Member, mconf: Dict[str, str], perms: str) -> Optional[bool]: + ... + + +def parse_core_permissions(channel: discord.TextChannel, member: discord.Member, mconf: Dict[str, str], perms: str) -> Optional[bool]: + """ + Parse permissions of a given TextChannel and Member, only parses core permissions (everyone,moderator,administrator,owner) and does not have verbosity + This is a lightweight alternative to parse_permissions for parsing simple permissions, while not sufficient for full command permission parsing. + + :returns: Optional[bool] - Has permission, or None if the perm name was not one of the core permissions + """ + + if perms == "everyone": + return True + elif perms == "moderator": + default_t = channel.permissions_for(member) + default = default_t.ban_members or default_t.administrator + modperm = (member, mconf["moderator-role"]) + adminperm = (member, mconf["admin-role"]) + return bool(default or _parse_role_perms(*modperm) or _parse_role_perms(*adminperm)) + elif perms == "administrator": + default = channel.permissions_for(member).administrator + adminperm = (member, mconf["admin-role"]) + return bool(default or _parse_role_perms(*adminperm)) + elif perms == "owner": + return bool(channel.guild.owner and member.id == channel.guild.owner.id) + + return None + + # Parse user permissions to run a command async def parse_permissions(message: discord.Message, mconf: Dict[str, str], perms: Permtype, verbose: bool = True) -> bool: """ - Parse the permissions of the given member object to check if they meet the required permtype + Parse the permissions of the given message object to check if they meet the required permtype Verbosity can be set to not print if the perm check failed :returns: bool @@ -317,24 +358,12 @@ async def parse_permissions(message: discord.Message, mconf: Dict[str, str], per return False you_shall_pass = False - if perms == "everyone": - you_shall_pass = True - elif perms == "moderator": - default = message.channel.permissions_for(message.author).ban_members - modperm = (message.author, mconf["moderator-role"]) - adminperm = (message.author, mconf["admin-role"]) - you_shall_pass = default or _parse_role_perms(*modperm) or _parse_role_perms(*adminperm) - elif perms == "administrator": - default = message.channel.permissions_for(message.author).administrator - adminperm = (message.author, mconf["admin-role"]) - you_shall_pass = default or _parse_role_perms(*adminperm) - elif perms == "owner": - # If we cant check the owner then skip it - if message.guild and message.guild.owner: - you_shall_pass = message.author.id == message.guild.owner.id - elif isinstance(perms, (tuple, list)): + if isinstance(perms, (tuple, list)): you_shall_pass = perms[1](message) perms = perms[0] + else: + # Cast None to False (previous behavior of parse_permissions) + you_shall_pass = parse_core_permissions(message.channel, message.author, mconf, perms) or False if you_shall_pass: return True From b62f9457f5adbe3833f32e175987559320ab6f89 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 28 Feb 2022 23:53:19 -0800 Subject: [PATCH 02/78] add more complex error message to parse_user_member (tests if channel/message/role id) --- libs/lib_parsers.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/libs/lib_parsers.py b/libs/lib_parsers.py index f47e98c..ed5858e 100644 --- a/libs/lib_parsers.py +++ b/libs/lib_parsers.py @@ -608,6 +608,39 @@ async def parse_channel_message(message: discord.Message, args: List[str], clien UserInterface = Union[discord.User, discord.Member] +# should return a union of many types but for now only handle discord.Message +async def _guess_id_type(message: discord.Message, mystery_id: int) -> Optional[Union[discord.Message, discord.Role, discord.TextChannel]]: + + # hot path current channel id + if message.channel.id == mystery_id and isinstance(message.channel, discord.TextChannel): + return message.channel + + # asserts guild + if not message.guild: + return None + + # requires guild + if (role := message.guild.get_role(mystery_id)) is not None: + return role + + if (chan := message.guild.get_channel(mystery_id)) is not None: + if isinstance(chan, discord.TextChannel): + return chan + + # asserts channel + if not isinstance(message.channel, discord.TextChannel): + return None + + # requires channel + try: + if (discord_message := await message.channel.fetch_message(mystery_id)) is not None: + return discord_message + except discord.errors.HTTPException: + pass + + return None + + async def parse_user_member_noexcept(message: discord.Message, args: List[str], client: discord.Client, @@ -643,6 +676,17 @@ async def parse_user_member_noexcept(message: discord.Message, if not (user := client.get_user(uid)): user = await client.fetch_user(uid) except (discord.errors.NotFound, discord.errors.HTTPException): + if (pot := await _guess_id_type(message, uid)) is not None: + errappend = "Note: While this ID is not a valid user ID, it is " + if isinstance(pot, discord.TextChannel): + errappend += f"a valid channel ID: <#{pot.id}>" + elif isinstance(pot, discord.Message): + errappend += f"a valid message by a user with ID {pot.author.id}\n(did you mean to select this user?)" + elif isinstance(pot, discord.Role): + errappend += "a valid role" + + raise lib_sonnetcommands.CommandError(f"User does not exist\n{errappend}") + raise lib_sonnetcommands.CommandError("User does not exist") return user, member From 5f9698a69b79e06a133ab74a09dfbd01716cb540 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 1 Mar 2022 00:11:31 -0800 Subject: [PATCH 03/78] add guild perm level to profile command --- cmds/cmd_utils.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 0fbe8d7..cf6cd5d 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -37,14 +37,14 @@ importlib.reload(lib_tparse) -from typing import Any, Final, List, Optional, Tuple, cast +from typing import Any, Final, List, Optional, Tuple, Dict, cast import lib_constants as constants import lib_lexdpyk_h as lexdpyk from lib_compatibility import discord_datetime_now, user_avatar_url from lib_db_obfuscator import db_hlapi from lib_loaders import datetime_now, embed_colors, load_embed_color -from lib_parsers import (parse_boolean, parse_permissions, parse_user_member_noexcept) +from lib_parsers import (parse_boolean, parse_permissions, parse_core_permissions, parse_user_member_noexcept) from lib_sonnetcommands import CallCtx, CommandCtx, SonnetCommand from lib_sonnetconfig import BOT_NAME from lib_tparse import Parser @@ -89,7 +89,19 @@ def parsedate(indata: Optional[datetime]) -> str: return "ERROR: Could not fetch this date" -async def profile_function(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +def _get_highest_perm(message: discord.Message, member: discord.Member, conf_cache: Dict[str, Any]) -> str: + if not isinstance(message.channel, discord.TextChannel): + return "everyone" + + highest = "everyone" + for i in ["moderator", "administrator", "owner"]: + if parse_core_permissions(message.channel, member, conf_cache, i): + highest = i + + return highest + + +async def profile_function(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: if not message.guild: return 1 @@ -105,7 +117,7 @@ async def profile_function(message: discord.Message, args: List[str], client: di "invisible": "\U000026AB (offline)" } - embed: Final = discord.Embed(title="User Information", description=f"User information for {user.mention}:", color=load_embed_color(message.guild, embed_colors.primary, kwargs["ramfs"])) + embed: Final = discord.Embed(title="User Information", description=f"User information for {user.mention}:", color=load_embed_color(message.guild, embed_colors.primary, ctx.ramfs)) embed.set_thumbnail(url=user_avatar_url(user)) embed.add_field(name="Username", value=str(user), inline=True) embed.add_field(name="User ID", value=str(user.id), inline=True) @@ -115,11 +127,12 @@ async def profile_function(message: discord.Message, args: List[str], client: di embed.add_field(name="Created", value=parsedate(user.created_at), inline=True) if member: embed.add_field(name="Joined", value=parsedate(member.joined_at), inline=True) + embed.add_field(name="Guild Perm Level", value=_get_highest_perm(message, member, ctx.conf_cache)) # Parse adding infraction count with db_hlapi(message.guild.id) as db: viewinfs = parse_boolean(db.grab_config("member-view-infractions") or "0") - moderator = await parse_permissions(message, kwargs["conf_cache"], "moderator", verbose=False) + moderator = await parse_permissions(message, ctx.conf_cache, "moderator", verbose=False) if moderator or (viewinfs and user.id == message.author.id): embed.add_field(name="Infractions", value=f"{db.grab_filter_infractions(user=user.id, count=True)}") From 727415fa9ab8b19041a9d38e8ecd82ad599e049a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 1 Mar 2022 08:40:23 -0800 Subject: [PATCH 04/78] bump version and add fast path for _grab_highest_perm --- cmds/cmd_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index cf6cd5d..85ef3fc 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -97,6 +97,8 @@ def _get_highest_perm(message: discord.Message, member: discord.Member, conf_cac for i in ["moderator", "administrator", "owner"]: if parse_core_permissions(message.channel, member, conf_cache, i): highest = i + else: + break return highest @@ -509,4 +511,4 @@ async def coinflip(message: discord.Message, args: List[str], client: discord.Cl } } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" From a28de9df868e5c43a2e745331899f4945230e544 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 1 Mar 2022 20:09:33 -0800 Subject: [PATCH 05/78] add KernelArgs and ToKernelArgs typesafe decorator --- libs/lib_lexdpyk_h.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/libs/lib_lexdpyk_h.py b/libs/lib_lexdpyk_h.py index 179d020..5f1b441 100644 --- a/libs/lib_lexdpyk_h.py +++ b/libs/lib_lexdpyk_h.py @@ -1,6 +1,8 @@ # Headers for the kernel ramfs import io +import discord +from dataclasses import dataclass from typing import Optional, List, Any, Tuple, Dict, Callable, Coroutine, Type, TypeVar, Protocol, overload @@ -64,3 +66,32 @@ class dlib_module(Protocol): dlib_modules_dict = Dict[str, Callable[..., Coroutine[Any, Any, None]]] + + +@dataclass +class KernelArgs: + """ + A wrapper around a kernels passed kwargs + """ + __slots__ = "kernel_version", "bot_start", "client", "ramfs", "kernel_ramfs", "command_modules", "dynamiclib_modules" + kernel_version: str + bot_start: float + client: discord.Client + ramfs: ram_filesystem + kernel_ramfs: ram_filesystem + command_modules: Tuple[List[cmd_module], cmd_modules_dict] + dynamiclib_modules: Tuple[List[dlib_module], dlib_modules_dict] + + +_FuncType = Callable[..., Coroutine[Any, Any, Any]] + + +def ToKernelArgs(f: _FuncType) -> _FuncType: + """ + A decorator to convert kwargs to KernelArgs for a kernel event handler + """ + def newfunc(*args: Any, **kwargs: Any) -> Coroutine[Any, Any, Any]: + nargs = (*args, KernelArgs(**kwargs)) + return f(*nargs) + + return newfunc From 63845519bc3f6b375e27dd64b3f241ee97df35c5 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 1 Mar 2022 20:09:56 -0800 Subject: [PATCH 06/78] add reference impl of KernelArgs in use --- dlibs/dlib_messages.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/dlibs/dlib_messages.py b/dlibs/dlib_messages.py index 9ea1076..65fb6dd 100644 --- a/dlibs/dlib_messages.py +++ b/dlibs/dlib_messages.py @@ -414,18 +414,13 @@ async def log_message_files(message: discord.Message, kernel_ramfs: lexdpyk.ram_ download_single_file(i, file_loc, key, iv, kernel_ramfs, [message.guild.id, message.id]) -async def on_message(message: discord.Message, **kargs: Any) -> None: +@lexdpyk.ToKernelArgs +async def on_message(message: discord.Message, kernel_args: lexdpyk.KernelArgs) -> None: - client: Final[discord.Client] = kargs["client"] - ramfs: Final[lexdpyk.ram_filesystem] = kargs["ramfs"] - kernel_ramfs: Final[lexdpyk.ram_filesystem] = kargs["kernel_ramfs"] - main_version_info: Final[str] = kargs["kernel_version"] - bot_start_time: Final[float] = kargs["bot_start"] - - command_modules: List[lexdpyk.cmd_module] - command_modules_dict: lexdpyk.cmd_modules_dict + client: Final = kernel_args.client + ramfs: Final = kernel_args.ramfs - command_modules, command_modules_dict = kargs["command_modules"] + command_modules, command_modules_dict = kernel_args.command_modules # Statistics. stats: Final[Dict[str, int]] = {"start": round(time.time() * 100000)} @@ -435,7 +430,7 @@ async def on_message(message: discord.Message, **kargs: Any) -> None: elif not message.guild: return - inc_statistics_better(message.guild.id, "on-message", kernel_ramfs) + inc_statistics_better(message.guild.id, "on-message", kernel_args.kernel_ramfs) # Load message conf stats["start-load-blacklist"] = round(time.time() * 100000) @@ -453,10 +448,10 @@ async def on_message(message: discord.Message, **kargs: Any) -> None: stats=stats, cmds=command_modules, ramfs=ramfs, - bot_start=bot_start_time, - dlibs=kargs["dynamiclib_modules"][0], - main_version=main_version_info, - kernel_ramfs=kargs["kernel_ramfs"], + bot_start=kernel_args.bot_start, + dlibs=kernel_args.dynamiclib_modules[0], + main_version=kernel_args.kernel_version, + kernel_ramfs=kernel_args.kernel_ramfs, conf_cache=mconf, verbose=True, cmds_dict=command_modules_dict, @@ -490,7 +485,7 @@ async def on_message(message: discord.Message, **kargs: Any) -> None: # Log files if not deleted if not message_deleted: - asyncio.create_task(log_message_files(message, kernel_ramfs)) + asyncio.create_task(log_message_files(message, kernel_args.kernel_ramfs)) # Check if this is meant for us. if not (message.content.startswith(mconf["prefix"])) or message_deleted: @@ -556,4 +551,4 @@ async def on_message(message: discord.Message, **kargs: Any) -> None: "on-message-delete": on_message_delete, } -version_info: Final = "1.2.12" +version_info: Final = "1.2.13-DEV" From 428325576cb6c941cd2555ff500cc80878c3eceb Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 4 Mar 2022 18:35:08 -0800 Subject: [PATCH 07/78] polish some old file logging code --- dlibs/dlib_messages.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/dlibs/dlib_messages.py b/dlibs/dlib_messages.py index 65fb6dd..10e2789 100644 --- a/dlibs/dlib_messages.py +++ b/dlibs/dlib_messages.py @@ -275,10 +275,10 @@ def antispam_check(message: discord.Message, ramfs: lexdpyk.ram_filesystem, anti if not message.guild: raise RuntimeError("How did we end up here? Basically antispam_check was called on a dm message, oops") - # Wierd behavior(ultrabear): message.created_at.timestamp() returns unaware dt so we need to use datetime.utcnow for timestamps in antispam + # Weird behavior(ultrabear): message.created_at.timestamp() returns unaware dt so we need to use datetime.utcnow for timestamps in antispam # Update(ultrabear): now that we use discord_datetime_now() we get an unaware dt or aware dt depending on dpy version - userid: Final = message.author.id + userid: Final[int] = message.author.id sent_at: Final[float] = message.created_at.timestamp() # Base antispam @@ -362,7 +362,7 @@ def antispam_check(message: discord.Message, ramfs: lexdpyk.ram_filesystem, anti return (False, "") -async def download_file(nfile: discord.Attachment, compression: Any, encryption: Any, filename: str, ramfs: lexdpyk.ram_filesystem, mgid: List[int]) -> None: +async def download_file(nfile: discord.Attachment, compression: Any, encryption: encrypted_writer, filename: str, ramfs: lexdpyk.ram_filesystem, guild_id: int, message_id: int) -> None: await nfile.save(compression, seek_begin=False) compression.close() @@ -375,20 +375,11 @@ async def download_file(nfile: discord.Attachment, compression: Any, encryption: except FileNotFoundError: pass try: - ramfs.rmdir(f"{mgid[0]}/files/{mgid[1]}") + ramfs.rmdir(f"{guild_id}/files/{message_id}") except FileNotFoundError: pass -def download_single_file(discord_file: discord.Attachment, filename: str, key: bytes, iv: bytes, ramfs: lexdpyk.ram_filesystem, mgid: List[int]) -> None: - - encryption_fileobj: Final = encrypted_writer(filename, key, iv) - - compression_fileobj: Final = lz4.frame.LZ4FrameFile(filename=encryption_fileobj, mode="wb") - - asyncio.create_task(download_file(discord_file, compression_fileobj, encryption_fileobj, filename, ramfs, mgid)) - - async def log_message_files(message: discord.Message, kernel_ramfs: lexdpyk.ram_filesystem) -> None: if not message.guild: return @@ -411,7 +402,12 @@ async def log_message_files(message: discord.Message, kernel_ramfs: lexdpyk.ram_ file_loc = f"./datastore/{message.guild.id}-{pointer}.cache.db" pointerfile.write(file_loc.encode("utf8")) - download_single_file(i, file_loc, key, iv, kernel_ramfs, [message.guild.id, message.id]) + # Create encryption and compression wrappers (raw -> compressed -> encrypted -> disk) + encryption_fileobj = encrypted_writer(file_loc, key, iv) + compression_fileobj = lz4.frame.LZ4FrameFile(filename=encryption_fileobj, mode="wb") + + # Do file downloading in async + asyncio.create_task(download_file(i, compression_fileobj, encryption_fileobj, file_loc, kernel_ramfs, message.guild.id, message.id)) @lexdpyk.ToKernelArgs From 824210459f9826ab0e554a22716f450f89043a3e Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 11 Mar 2022 20:05:58 -0800 Subject: [PATCH 08/78] make grab-message check for read message perms and allow everyone perm to use --- cmds/cmd_moderation.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index b21558d..654fd27 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -706,11 +706,19 @@ async def grab_guild_message(message: discord.Message, args: List[str], client: if not message.guild: return 1 + discord_message: discord.Message + discord_message, nargs = await parse_channel_message_noexcept(message, args, client) - if not discord_message.guild: - await message.channel.send("ERROR: Message not in any guild") - return 1 + if not discord_message.guild or isinstance(discord_message.channel, (discord.DMChannel, discord.GroupChannel)): + raise lib_sonnetcommands.CommandError("ERROR: Message not in any guild") + + if not isinstance(message.author, discord.Member): + raise lib_sonnetcommands.CommandError("ERROR: The user that ran this command is no longer in the guild?") + + # do extra validation that they can see this message + if not discord_message.channel.permissions_for(message.author).read_messages: + raise lib_sonnetcommands.CommandError("ERROR: You do not have permission to view this message") sendraw = False for arg in args[nargs:]: @@ -731,7 +739,7 @@ async def grab_guild_message(message: discord.Message, args: List[str], client: fileobjs = grab_files(discord_message.guild.id, discord_message.id, ctx.kernel_ramfs) # Grab files async if not in cache - if not fileobjs: + if fileobjs is None: awaitobjs = [asyncio.create_task(i.to_file()) for i in discord_message.attachments] fileobjs = [await i for i in awaitobjs] @@ -745,8 +753,7 @@ async def grab_guild_message(message: discord.Message, args: List[str], client: try: await message.channel.send("There were files attached but they exceeded the guild filesize limit", embed=message_embed) except discord.errors.Forbidden: - await message.channel.send(constants.sonnet.error_embed) - return 1 + raise lib_sonnetcommands.CommandError(constants.sonnet.error_embed) return 0 @@ -965,13 +972,11 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco 'get-message': { 'alias': 'grab-message' }, - 'grab-message': - { - 'pretty_name': 'grab-message [-r]', - 'description': 'Grab a message and show its contents, specify -r to get message content as a file', - 'permission': 'moderator', - 'execute': grab_guild_message - }, + 'grab-message': { + 'pretty_name': 'grab-message [-r]', + 'description': 'Grab a message and show its contents, specify -r to get message content as a file', + 'execute': grab_guild_message + }, 'purge': { 'pretty_name': 'purge [user]', From 3de0e30609588de66483e954098370523e2ff6f8 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 11 Mar 2022 20:09:33 -0800 Subject: [PATCH 09/78] move grab-message from moderation to utils --- cmds/cmd_moderation.py | 66 +----------------------------------------- cmds/cmd_utils.py | 66 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index 654fd27..d570740 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -37,7 +37,7 @@ from lib_goparsers import MustParseDuration from lib_loaders import generate_infractionid, load_embed_color, embed_colors, datetime_now, datetime_unix from lib_db_obfuscator import db_hlapi -from lib_parsers import grab_files, generate_reply_field, parse_channel_message_noexcept, parse_user_member, format_duration, paginate_noexcept +from lib_parsers import parse_user_member, format_duration, paginate_noexcept from lib_compatibility import user_avatar_url from lib_sonnetconfig import BOT_NAME, REGEX_VERSION from lib_sonnetcommands import CommandCtx @@ -702,62 +702,6 @@ async def delete_infraction(message: discord.Message, args: List[str], client: d return 1 -async def grab_guild_message(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - discord_message: discord.Message - - discord_message, nargs = await parse_channel_message_noexcept(message, args, client) - - if not discord_message.guild or isinstance(discord_message.channel, (discord.DMChannel, discord.GroupChannel)): - raise lib_sonnetcommands.CommandError("ERROR: Message not in any guild") - - if not isinstance(message.author, discord.Member): - raise lib_sonnetcommands.CommandError("ERROR: The user that ran this command is no longer in the guild?") - - # do extra validation that they can see this message - if not discord_message.channel.permissions_for(message.author).read_messages: - raise lib_sonnetcommands.CommandError("ERROR: You do not have permission to view this message") - - sendraw = False - for arg in args[nargs:]: - if arg in ["-r", "--raw"]: - sendraw = True - break - - # Generate replies - message_content = generate_reply_field(discord_message) - - # Message has been grabbed, start generating embed - message_embed = discord.Embed(title=f"Message in #{discord_message.channel}", description=message_content, color=load_embed_color(message.guild, embed_colors.primary, ctx.ramfs)) - - message_embed.set_author(name=str(discord_message.author), icon_url=user_avatar_url(discord_message.author)) - message_embed.timestamp = discord_message.created_at - - # Grab files from cache - fileobjs = grab_files(discord_message.guild.id, discord_message.id, ctx.kernel_ramfs) - - # Grab files async if not in cache - if fileobjs is None: - awaitobjs = [asyncio.create_task(i.to_file()) for i in discord_message.attachments] - fileobjs = [await i for i in awaitobjs] - - if sendraw: - file_content = io.BytesIO(discord_message.content.encode("utf8")) - fileobjs.append(discord.File(file_content, filename=f"{discord_message.id}.at.{int(datetime_now().timestamp())}.txt")) - - try: - await message.channel.send(embed=message_embed, files=fileobjs) - except discord.errors.HTTPException: - try: - await message.channel.send("There were files attached but they exceeded the guild filesize limit", embed=message_embed) - except discord.errors.Forbidden: - raise lib_sonnetcommands.CommandError(constants.sonnet.error_embed) - - return 0 - - class purger: __slots__ = "user_id", @@ -969,14 +913,6 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco 'permission': 'administrator', 'execute': delete_infraction }, - 'get-message': { - 'alias': 'grab-message' - }, - 'grab-message': { - 'pretty_name': 'grab-message [-r]', - 'description': 'Grab a message and show its contents, specify -r to get message content as a file', - 'execute': grab_guild_message - }, 'purge': { 'pretty_name': 'purge [user]', diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 85ef3fc..9015867 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -44,7 +44,7 @@ from lib_compatibility import discord_datetime_now, user_avatar_url from lib_db_obfuscator import db_hlapi from lib_loaders import datetime_now, embed_colors, load_embed_color -from lib_parsers import (parse_boolean, parse_permissions, parse_core_permissions, parse_user_member_noexcept) +from lib_parsers import (parse_boolean, parse_permissions, parse_core_permissions, parse_user_member_noexcept, parse_channel_message_noexcept, generate_reply_field, grab_files) from lib_sonnetcommands import CallCtx, CommandCtx, SonnetCommand from lib_sonnetconfig import BOT_NAME from lib_tparse import Parser @@ -411,6 +411,62 @@ async def grab_role_info(message: discord.Message, args: List[str], client: disc raise lib_sonnetcommands.CommandError("ERROR: Could not grab role from this guild") +async def grab_guild_message(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + discord_message: discord.Message + + discord_message, nargs = await parse_channel_message_noexcept(message, args, client) + + if not discord_message.guild or isinstance(discord_message.channel, (discord.DMChannel, discord.GroupChannel)): + raise lib_sonnetcommands.CommandError("ERROR: Message not in any guild") + + if not isinstance(message.author, discord.Member): + raise lib_sonnetcommands.CommandError("ERROR: The user that ran this command is no longer in the guild?") + + # do extra validation that they can see this message + if not discord_message.channel.permissions_for(message.author).read_messages: + raise lib_sonnetcommands.CommandError("ERROR: You do not have permission to view this message") + + sendraw = False + for arg in args[nargs:]: + if arg in ["-r", "--raw"]: + sendraw = True + break + + # Generate replies + message_content = generate_reply_field(discord_message) + + # Message has been grabbed, start generating embed + message_embed = discord.Embed(title=f"Message in #{discord_message.channel}", description=message_content, color=load_embed_color(message.guild, embed_colors.primary, ctx.ramfs)) + + message_embed.set_author(name=str(discord_message.author), icon_url=user_avatar_url(discord_message.author)) + message_embed.timestamp = discord_message.created_at + + # Grab files from cache + fileobjs = grab_files(discord_message.guild.id, discord_message.id, ctx.kernel_ramfs) + + # Grab files async if not in cache + if fileobjs is None: + awaitobjs = [asyncio.create_task(i.to_file()) for i in discord_message.attachments] + fileobjs = [await i for i in awaitobjs] + + if sendraw: + file_content = io.BytesIO(discord_message.content.encode("utf8")) + fileobjs.append(discord.File(file_content, filename=f"{discord_message.id}.at.{int(datetime_now().timestamp())}.txt")) + + try: + await message.channel.send(embed=message_embed, files=fileobjs) + except discord.errors.HTTPException: + try: + await message.channel.send("There were files attached but they exceeded the guild filesize limit", embed=message_embed) + except discord.errors.Forbidden: + raise lib_sonnetcommands.CommandError(constants.sonnet.error_embed) + + return 0 + + async def initialise_poll(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> None: try: @@ -495,6 +551,14 @@ async def coinflip(message: discord.Message, args: List[str], client: discord.Cl 'cache': 'keep', 'execute': grab_guild_info }, + 'get-message': { + 'alias': 'grab-message' + }, + 'grab-message': { + 'pretty_name': 'grab-message [-r]', + 'description': 'Grab a message and show its contents, specify -r to get message content as a file', + 'execute': grab_guild_message + }, 'poll': { 'pretty_name': 'poll', 'description': 'Start a reaction based poll on the message', From 29a69f2762b483aa38d8b2e783c5ee5bf5ee9e43 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 11 Mar 2022 20:13:11 -0800 Subject: [PATCH 10/78] version bump moderation module --- cmds/cmd_moderation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index d570740..914cceb 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -923,4 +923,4 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco } } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" From 534971f6e44bb7227aca1f572dd25fd9af7de6aa Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 11 Mar 2022 20:59:54 -0800 Subject: [PATCH 11/78] add _ to url content capture targets --- libs/lib_starboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/lib_starboard.py b/libs/lib_starboard.py index f42b89e..0625ec1 100644 --- a/libs/lib_starboard.py +++ b/libs/lib_starboard.py @@ -30,7 +30,7 @@ globals()["re"] = importlib.import_module(REGEX_VERSION) _image_filetypes = [".png", ".bmp", ".jpg", ".jpeg", ".gif", ".webp"] -_url_chars = "[a-zA-Z0-9\.\-]" +_url_chars = "[a-zA-Z0-9\.\-_]" # match https?:// # match optional newline From c92574ed5e2b306728812fb2093356242b027a4d Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 16 Mar 2022 09:14:52 -0700 Subject: [PATCH 12/78] add sub command --- cmds/cmd_scripting.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 773ceb5..0575417 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -290,6 +290,31 @@ async def sonnet_async_map(message: discord.Message, args: List[str], client: di if ctx.verbose: await message.channel.send(f"Completed execution of {len(targs)} instances of {command} in {fmttime}ms") +async def run_as_subcommand(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: + + # command check, perm check, run + + if args: + command = args[0] + + if command not in ctx.cmds_dict: + raise lib_sonnetcommands.CommandError("ERROR: Command does not exist") + + sonnetc = SonnetCommand(ctx.cmds_dict[command], ctx.cmds_dict) + + if not await parse_permissions(message, ctx.conf_cache, sonnetc.permission): + return 1 + + # set to subcommand + newctx = pycopy.copy(ctx) + newctx.verbose = False + + return await sonnetc.execute_ctx(message, args[1:], client, newctx) + + else: + raise lib_sonnetcommands.CommandError("ERROR: No command specified") + + category_info = {'name': 'scripting', 'pretty_name': 'Scripting', 'description': 'Scripting tools for all your shell like needs'} commands = { @@ -325,7 +350,12 @@ async def sonnet_async_map(message: discord.Message, args: List[str], client: di 'permission': 'moderator', 'cache': 'keep', 'execute': sonnet_async_map - } + }, + 'sub': { + 'pretty_name': 'sub [args]+', + 'description': 'runs a command as a subcommand', + 'execute': run_as_subcommand, + } } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" From e8efa890de6679e75b78ce795fb3a2d36f228537 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 16 Mar 2022 22:39:09 -0700 Subject: [PATCH 13/78] add sleep command --- cmds/cmd_scripting.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 0575417..ff15ced 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -315,6 +315,24 @@ async def run_as_subcommand(message: discord.Message, args: List[str], client: d raise lib_sonnetcommands.CommandError("ERROR: No command specified") +async def sleep_for(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> None: + + if ctx.verbose: + raise lib_sonnetcommands.CommandError("ERROR: Can only run sleep as a subcommand") + + try: + sleep_time = float(args[0]) + except IndexError: + raise lib_sonnetcommands.CommandError("ERROR: No sleep time specified") + except ValueError: + raise lib_sonnetcommands.CommandError("ERROR: Could not parse sleep duration") + + if not (0 <= sleep_time <= 30): + raise lib_sonnetcommands.CommandError("ERROR: Cannot sleep for more than 30 seconds or less than 0 seconds") + + await asyncio.sleep(sleep_time) + + category_info = {'name': 'scripting', 'pretty_name': 'Scripting', 'description': 'Scripting tools for all your shell like needs'} commands = { @@ -355,7 +373,13 @@ async def run_as_subcommand(message: discord.Message, args: List[str], client: d 'pretty_name': 'sub [args]+', 'description': 'runs a command as a subcommand', 'execute': run_as_subcommand, - } + }, + 'sleep': { + 'pretty_name': 'sleep ', + 'description': 'Suspends execution for up to 30 seconds, for use in map/sonnetsh', + 'permission': 'moderator', + 'execute': sleep_for, + }, } version_info: str = "1.2.13-DEV" From ffd222cccec479f4f2e64935a6a26b7b2c8852de Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 16 Mar 2022 22:42:11 -0700 Subject: [PATCH 14/78] improve (a)map error strings --- cmds/cmd_scripting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index ff15ced..20a97e8 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -152,7 +152,7 @@ async def map_preprocessor(message: discord.Message, args: List[str], client: di raise MapProcessError("ERRNO") if not targs: - await message.channel.send("No command specified") + await message.channel.send("ERROR: No command specified") raise MapProcessError("ERRNO") # parses instances of -startargs and -endargs @@ -165,17 +165,17 @@ async def map_preprocessor(message: discord.Message, args: List[str], client: di else: exargs[1].extend(targs.pop(0).split()) except IndexError: - await message.channel.send("-s/-e specified but no input") + await message.channel.send("ERROR: -s/-e specified but no input") raise MapProcessError("ERRNO") if not targs: - await message.channel.send("No command specified") + await message.channel.send("ERROR: No command specified") raise MapProcessError("ERRNO") command = targs.pop(0) if command not in cmds_dict: - await message.channel.send("Invalid command") + await message.channel.send("ERROR: Command not found") raise MapProcessError("ERRNO") cmd = SonnetCommand(cmds_dict[command], cmds_dict) From 3eead7626c2cb4b95187f4d48058c7f8b43e8dcc Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 17 Mar 2022 09:09:18 -0700 Subject: [PATCH 15/78] limit sub/map to not allow selfcalls, add cname to error messages --- cmds/cmd_scripting.py | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 20a97e8..4769e3d 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -142,18 +142,16 @@ class MapProcessError(Exception): __slots__ = () -async def map_preprocessor(message: discord.Message, args: List[str], client: discord.Client, cmds_dict: lexdpyk.cmd_modules_dict, - conf_cache: Dict[str, Any]) -> Tuple[List[str], SonnetCommand, str, Tuple[List[str], List[str]]]: +async def map_preprocessor_someexcept(message: discord.Message, args: List[str], client: discord.Client, cmds_dict: lexdpyk.cmd_modules_dict, conf_cache: Dict[str, Any], + cname: str) -> Tuple[List[str], SonnetCommand, str, Tuple[List[str], List[str]]]: try: targs: List[str] = shlex.split(" ".join(args)) except ValueError: - await message.channel.send("ERROR: shlex parser could not parse args") - raise MapProcessError("ERRNO") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): shlex parser could not parse args") if not targs: - await message.channel.send("ERROR: No command specified") - raise MapProcessError("ERRNO") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): No command specified") # parses instances of -startargs and -endargs exargs: Tuple[List[str], List[str]] = ([], []) @@ -165,27 +163,26 @@ async def map_preprocessor(message: discord.Message, args: List[str], client: di else: exargs[1].extend(targs.pop(0).split()) except IndexError: - await message.channel.send("ERROR: -s/-e specified but no input") - raise MapProcessError("ERRNO") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): -s/-e specified but no input") if not targs: - await message.channel.send("ERROR: No command specified") - raise MapProcessError("ERRNO") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): No command specified") command = targs.pop(0) if command not in cmds_dict: - await message.channel.send("ERROR: Command not found") - raise MapProcessError("ERRNO") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): Command not found") cmd = SonnetCommand(cmds_dict[command], cmds_dict) if not await parse_permissions(message, conf_cache, cmd.permission): raise MapProcessError("ERRNO") + if cmd.execute_ctx == sonnet_map or cmd.execute_ctx == sonnet_async_map: + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): Cannot call map/amap from {cname}") + if len(targs) > exec_lim: - await message.channel.send(f"ERROR: Exceeded limit of {exec_lim} iterations") - raise MapProcessError("ERR LIM EXEEDED") + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): Exceeded limit of {exec_lim} iterations") return targs, cmd, command, exargs @@ -198,7 +195,7 @@ async def sonnet_map(message: discord.Message, args: List[str], client: discord. cmds_dict = ctx.cmds_dict try: - targs, cmd, command, exargs = await map_preprocessor(message, args, client, cmds_dict, ctx.conf_cache) + targs, cmd, command, exargs = await map_preprocessor_someexcept(message, args, client, cmds_dict, ctx.conf_cache, "map") except MapProcessError: return 1 @@ -222,7 +219,7 @@ async def sonnet_map(message: discord.Message, args: List[str], client: discord. suc = 1 if suc != 0: - await message.channel.send(f"ERROR: command `{command}` exited with non success status") + await message.channel.send(f"ERROR(map): command `{command}` exited with non success status") return 1 # Do cache sweep on command @@ -256,7 +253,7 @@ async def sonnet_async_map(message: discord.Message, args: List[str], client: di cmds_dict: lexdpyk.cmd_modules_dict = ctx.cmds_dict try: - targs, cmd, command, exargs = await map_preprocessor(message, args, client, cmds_dict, ctx.conf_cache) + targs, cmd, command, exargs = await map_preprocessor_someexcept(message, args, client, cmds_dict, ctx.conf_cache, "amap") except MapProcessError: return 1 @@ -298,10 +295,13 @@ async def run_as_subcommand(message: discord.Message, args: List[str], client: d command = args[0] if command not in ctx.cmds_dict: - raise lib_sonnetcommands.CommandError("ERROR: Command does not exist") + raise lib_sonnetcommands.CommandError("ERROR(sub): Command does not exist") sonnetc = SonnetCommand(ctx.cmds_dict[command], ctx.cmds_dict) + if sonnetc.execute_ctx == run_as_subcommand: + raise lib_sonnetcommands.CommandError("ERROR(sub): Cannot call sub from sub") + if not await parse_permissions(message, ctx.conf_cache, sonnetc.permission): return 1 @@ -312,23 +312,23 @@ async def run_as_subcommand(message: discord.Message, args: List[str], client: d return await sonnetc.execute_ctx(message, args[1:], client, newctx) else: - raise lib_sonnetcommands.CommandError("ERROR: No command specified") + raise lib_sonnetcommands.CommandError("ERROR(sub): No command specified") async def sleep_for(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> None: if ctx.verbose: - raise lib_sonnetcommands.CommandError("ERROR: Can only run sleep as a subcommand") + raise lib_sonnetcommands.CommandError("ERROR(sleep): Can only run sleep as a subcommand") try: sleep_time = float(args[0]) except IndexError: - raise lib_sonnetcommands.CommandError("ERROR: No sleep time specified") + raise lib_sonnetcommands.CommandError("ERROR(sleep): No sleep time specified") except ValueError: - raise lib_sonnetcommands.CommandError("ERROR: Could not parse sleep duration") + raise lib_sonnetcommands.CommandError("ERROR(sleep): Could not parse sleep duration") if not (0 <= sleep_time <= 30): - raise lib_sonnetcommands.CommandError("ERROR: Cannot sleep for more than 30 seconds or less than 0 seconds") + raise lib_sonnetcommands.CommandError("ERROR(sleep): Cannot sleep for more than 30 seconds or less than 0 seconds") await asyncio.sleep(sleep_time) From a9c732952e84f3501971f90968130048cf65d2af Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 17 Mar 2022 09:29:15 -0700 Subject: [PATCH 16/78] make full help sort command names --- cmds/cmd_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 9015867..0b95a9a 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -252,7 +252,7 @@ async def full_help(self, page: int, per_page: int) -> discord.Embed: for module in sorted(cmds, key=lambda m: m.category_info['pretty_name'])[(page * per_page):(page * per_page) + per_page]: mnames = [f"`{i}`" for i in module.commands if 'alias' not in module.commands[i]] - helptext = ', '.join(mnames) if mnames else module.category_info['description'] + helptext = ', '.join(sorted(mnames)) if mnames else module.category_info['description'] cmd_embed.add_field(name=f"{module.category_info['pretty_name']} ({module.category_info['name']})", value=helptext, inline=False) cmd_embed.set_footer(text=f"Total Commands: {total} | Total Endpoints: {len(cmds_dict)}") From d2804fc833188a02edfbebdbb11f795501599e39 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 17 Mar 2022 13:06:00 -0700 Subject: [PATCH 17/78] add softban command --- cmds/cmd_moderation.py | 58 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index 914cceb..8d0e8c8 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -348,7 +348,7 @@ async def ban_user(message: discord.Message, args: List[str], client: discord.Cl await message.channel.send(f"{BOT_NAME} does not have permission to ban this user.") return 1 - delete_str = f", and deleted {delete_days} day{'s'*(delete_days!=1)} of messages," * bool(delete_days) + delete_str = f", and deleted {delete_days} day{'s'*(delete_days!=1)} of messages," if delete_days else "" mod_str = f" with {','.join(m.title for m in modifiers)}" if modifiers else "" if ctx.verbose: await message.channel.send(f"Banned {user.mention} with ID {user.id}{mod_str}{delete_str} for {reason}", allowed_mentions=discord.AllowedMentions.none()) @@ -375,13 +375,60 @@ async def unban_user(message: discord.Message, args: List[str], client: discord. await message.channel.send(f"{BOT_NAME} does not have permission to unban this user.") return 1 except discord.errors.NotFound: - await message.channel.send("This user is not banned") + await message.channel.send("This user is not banned.") return 1 if verbose: await message.channel.send(f"Unbanned {user.mention} with ID {user.id} for {reason}", allowed_mentions=discord.AllowedMentions.none()) return 0 +# bans and unbans a user, idk +async def softban_user(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + modifiers = parse_infraction_modifiers(message.guild, args) + + if len(args) >= 3 and args[1] in ["-d", "--days"]: + try: + delete_days = int(args[2]) + del args[2] + del args[1] + except ValueError: + delete_days = 0 + else: + delete_days = 0 + + # bounds check (docs say 0 is min and 7 is max) + if delete_days > 7: delete_days = 7 + elif delete_days < 0: delete_days = 0 + + try: + member, user, reason, _, dm_sent = await process_infraction(message, args, client, "softban", ctx.ramfs, automod=ctx.automod, modifiers=modifiers) + except InfractionGenerationError: + return 1 + + try: + if member and dm_sent: + await dm_sent # Wait for dm to be sent before banning + await message.guild.ban(user, delete_message_days=delete_days, reason=reason[:512]) + except discord.errors.Forbidden: + raise lib_sonnetcommands.CommandError(f"{BOT_NAME} does not have permission to ban this user.") + + try: + await message.guild.unban(user, reason=reason[:512]) + except discord.errors.Forbidden: + raise lib_sonnetcommands.CommandError(f"{BOT_NAME} does not have permission to unban this user.") + except discord.errors.NotFound: + raise lib_sonnetcommands.CommandError(f"Unbanning failed: User is not banned.\n(Maybe user was unbanned before {BOT_NAME} could?)\n(Maybe discord did not register the ban properly?)") + + delete_str = f", and deleted {delete_days} day{'s'*(delete_days!=1)} of messages," if delete_days else "" + mod_str = f" with {','.join(m.title for m in modifiers)}" if modifiers else "" + + if ctx.verbose: await message.channel.send(f"Softbanned {user.mention} with ID {user.id}{mod_str}{delete_str} for {reason}", allowed_mentions=discord.AllowedMentions.none()) + return 0 + + class NoMuteRole(Exception): __slots__ = () @@ -860,6 +907,13 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco 'permission': 'moderator', 'execute': unban_user }, + 'softban': + { + 'pretty_name': 'softban [+modifiers] [-d DAYS] [reason]', + 'description': 'Softban (ban and then immediately unban) a user, optionally delete messages with -d', + 'permission': 'moderator', + 'execute': softban_user, + }, 'mute': { 'pretty_name': 'mute [+modifiers] [time[h|m|S]] [reason]', 'description': 'Mute a user, defaults to no unmute (0s)', From 2a63a198a81a2395f3bb2adf17d3f706efd20532 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 17 Mar 2022 13:28:59 -0700 Subject: [PATCH 18/78] split cmd_moderation into moderation and bookkeeping --- cmds/cmd_bookkeeping.py | 343 ++++++++++++++++++++++++++++++++++++++++ cmds/cmd_moderation.py | 309 +----------------------------------- 2 files changed, 347 insertions(+), 305 deletions(-) create mode 100644 cmds/cmd_bookkeeping.py diff --git a/cmds/cmd_bookkeeping.py b/cmds/cmd_bookkeeping.py new file mode 100644 index 0000000..0e45946 --- /dev/null +++ b/cmds/cmd_bookkeeping.py @@ -0,0 +1,343 @@ +# Moderation commands that focus on managing rather than immediate action +# Ultrabear 2022 + +import importlib + +import discord, time, math, io, shlex + +import lib_db_obfuscator + +importlib.reload(lib_db_obfuscator) +import lib_loaders + +importlib.reload(lib_loaders) +import lib_parsers + +importlib.reload(lib_parsers) +import lib_constants + +importlib.reload(lib_constants) +import lib_sonnetconfig + +importlib.reload(lib_sonnetconfig) +import lib_sonnetcommands + +importlib.reload(lib_sonnetcommands) +import lib_tparse + +importlib.reload(lib_tparse) + +from lib_loaders import load_embed_color, embed_colors, datetime_now, datetime_unix +from lib_db_obfuscator import db_hlapi +from lib_parsers import format_duration, paginate_noexcept +from lib_sonnetconfig import REGEX_VERSION +from lib_sonnetcommands import CommandCtx +from lib_tparse import Parser +import lib_constants as constants + +from typing import List, Tuple, Optional + +# Import re to trick type checker into using re stubs +import re + +# Import into globals hashmap to ignore pyflakes redefinition errors +globals()["re"] = importlib.import_module(REGEX_VERSION) + + +def get_user_id(s: str) -> int: + return int(s.strip("<@!>")) + + +async def search_infractions_by_user(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + tstart = time.monotonic() + + if not ctx.verbose: + raise lib_sonnetcommands.CommandError("ERROR: search-infractions only meant to be called directly") + + # Reparse args + try: + args = shlex.split(" ".join(args)) + except ValueError: + raise lib_sonnetcommands.CommandError("ERROR: Shlex failed to parse arguments") + + # Parse flags + parser = Parser("search-infractions") + + selected_chunk_f = parser.add_arg(["-p", "--page"], lambda s: int(s) - 1) + responsible_mod_f = parser.add_arg(["-m", "--mod"], get_user_id) + user_affected_f = parser.add_arg(["-u", "--user"], get_user_id) + per_page_f = parser.add_arg(["-i", "--infractioncount"], int) + infraction_type_f = parser.add_arg(["-t", "--type"], str) + filtering_f = parser.add_arg(["-f", "--filter"], str) + automod_f = lib_tparse.add_true_false_flag(parser, "automod") + + try: + parser.parse(args, stderr=io.StringIO(), exit_on_fail=False, lazy=True) + except lib_tparse.ParseFailureError: + await message.channel.send("Failed to parse flags") + return 1 + + selected_chunk = selected_chunk_f.get(0) + per_page = per_page_f.get(20) + user_affected = user_affected_f.get() + + # Default to user if no user/mod flags are supplied + if None is responsible_mod_f.get() is user_affected: + try: + user_affected = get_user_id(args[0]) + except (IndexError, ValueError): + pass + + if not 1 <= per_page <= 40: # pytype: disable=unsupported-operands + await message.channel.send("ERROR: Cannot exceed range 1-40 infractions per page") + return 1 + + refilter: "Optional[re.Pattern[str]]" + + if (f := filtering_f.get()) is not None: + try: + refilter = re.compile(f) + except re.error: + raise lib_sonnetcommands.CommandError("ERROR: Filter regex is invalid") + else: + refilter = None + + with db_hlapi(message.guild.id) as db: + if user_affected or responsible_mod_f.get(): + infractions = db.grab_filter_infractions(user=user_affected, moderator=responsible_mod_f.get(), itype=infraction_type_f.get(), automod=automod_f.get()) + assert isinstance(infractions, list) + else: + await message.channel.send("Please specify a user or moderator") + return 1 + + if refilter is not None: + infractions = [i for i in infractions if refilter.findall(i[4])] + + # Sort newest first + infractions.sort(reverse=True, key=lambda a: a[5]) + + # Return if no infractions, this is not an error as it returned a valid status + if not infractions: + await message.channel.send("No infractions found") + return 0 + + cpagecount = math.ceil(len(infractions) / per_page) + + # Test if valid page + if selected_chunk == -1: # ik it says page 0 but it does -1 on user input so the user would have entered 0 + raise lib_sonnetcommands.CommandError("ERROR: Cannot go to page 0") + elif selected_chunk < -1: + selected_chunk = (cpagecount + selected_chunk) + 1 + + def format_infraction(i: Tuple[str, str, str, str, str, int]) -> str: + return ', '.join([i[0], i[3], i[4]]) + + page = paginate_noexcept(infractions, selected_chunk, per_page, 1900, fmtfunc=format_infraction) + + tprint = (time.monotonic() - tstart) * 1000 + + await message.channel.send(f"Page {selected_chunk+1} / {cpagecount} ({len(infractions)} infraction{'s'*(len(infractions)!=1)}) ({tprint:.1f}ms)\n```css\nID, Type, Reason\n{page}```") + return 0 + + +async def get_detailed_infraction(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + if args: + with db_hlapi(message.guild.id) as db: + infraction = db.grab_infraction(args[0]) + if not infraction: + await message.channel.send("ERROR: Infraction ID does not exist") + return 1 + else: + await message.channel.send("ERROR: No argument supplied") + return 1 + + # Unpack this nightmare lmao + # pylint: disable=E0633 + infraction_id, user_id, moderator_id, infraction_type, reason, timestamp = infraction + + infraction_embed = discord.Embed(title="Infraction Search", description=f"Infraction for <@{user_id}>:", color=load_embed_color(message.guild, embed_colors.primary, ctx.ramfs)) + infraction_embed.add_field(name="Infraction ID", value=infraction_id) + infraction_embed.add_field(name="Moderator", value=f"<@{moderator_id}>") + infraction_embed.add_field(name="Type", value=infraction_type) + infraction_embed.add_field(name="Reason", value=reason) + + infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") + infraction_embed.timestamp = datetime_unix(int(timestamp)) + + try: + await message.channel.send(embed=infraction_embed) + return 0 + except discord.errors.Forbidden: + await message.channel.send(constants.sonnet.error_embed) + return 1 + + +async def delete_infraction(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + if args: + with db_hlapi(message.guild.id) as db: + infraction = db.grab_infraction(args[0]) + if not infraction: + await message.channel.send("ERROR: Infraction ID does not exist") + return 1 + # pylint: disable=E1136 + db.delete_infraction(infraction[0]) + else: + await message.channel.send("ERROR: No argument supplied") + return 1 + + if not ctx.verbose: + return 0 + + # pylint: disable=E0633 + infraction_id, user_id, moderator_id, infraction_type, reason, timestamp = infraction + + infraction_embed = discord.Embed(title="Infraction Deleted", description=f"Infraction for <@{user_id}>:", color=load_embed_color(message.guild, embed_colors.deletion, ctx.ramfs)) + infraction_embed.add_field(name="Infraction ID", value=infraction_id) + infraction_embed.add_field(name="Moderator", value=f"<@{moderator_id}>") + infraction_embed.add_field(name="Type", value=infraction_type) + infraction_embed.add_field(name="Reason", value=reason) + + infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") + + infraction_embed.timestamp = datetime_unix(int(timestamp)) + + try: + await message.channel.send(embed=infraction_embed) + return 0 + except discord.errors.Forbidden: + await message.channel.send(constants.sonnet.error_embed) + return 1 + + +def notneg(v: int) -> int: + if v < 0: raise ValueError + return v + + +async def query_mutedb(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + parser = Parser("query_mutedb") + pageP = parser.add_arg(["-p", "--page"], lambda s: notneg(int(s) - 1)) + + try: + parser.parse(args, stderr=io.StringIO(), exit_on_fail=False, lazy=True) + except lib_tparse.ParseFailureError: + raise lib_sonnetcommands.CommandError("Failed to parse page") + + page = pageP.get(0) + + per_page = 10 + + with db_hlapi(message.guild.id) as db: + table: List[Tuple[str, str, int]] = db.fetch_guild_mutes() + + if not table: + await message.channel.send("No Muted users in database") + return 0 + + def fmtfunc(v: Tuple[str, str, int]) -> str: + ts = "No Unmute" if v[2] == 0 else format_duration(v[2] - datetime_now().timestamp()) + return (f"{v[1]}, {v[0]}, {ts}") + + out = paginate_noexcept(sorted(table, key=lambda i: i[2]), page, per_page, 1500, fmtfunc) + + await message.channel.send(f"Page {page+1} / {len(table)//per_page+1}, ({len(table)} mute{'s'*(len(table)!=1)})```css\nUid, InfractionID, Unmuted in\n{out}```") + return 0 + + +async def remove_mutedb(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + try: + uid = int(args[0].strip("<@!>")) + except ValueError: + await message.channel.send("ERROR: Invalid user") + return 1 + except IndexError: + await message.channel.send("ERROR: No user specified") + return 1 + + with db_hlapi(message.guild.id) as db: + if db.is_muted(userid=uid): + db.unmute_user(userid=uid) + + await message.channel.send("Removed user from mute database") + + else: + await message.channel.send("ERROR: User is not in mute database") + return 1 + + return 0 + + +category_info = {'name': 'bookkeeping', 'pretty_name': 'Bookkeeping', 'description': 'Commands to assist with moderation bookkeeping'} + +commands = { + 'remove-mute': { + 'pretty_name': 'remove-mute ', + 'description': 'Removes a user from the mute database. Does not unmute in guild', + 'permission': 'administrator', + 'execute': remove_mutedb, + }, + 'list-mutes': { + 'pretty_name': 'list-mutes [-p PAGE]', + 'description': 'List all mutes in the mute database', + 'permission': 'moderator', + 'execute': query_mutedb, + }, + 'warnings': { + 'alias': 'search-infractions' + }, + 'list-infractions': { + 'alias': 'search-infractions' + }, + 'infractions': { + 'alias': 'search-infractions' + }, + 'search-infractions': + { + 'pretty_name': 'search-infractions <-u USER | -m MOD> [-t TYPE] [-p PAGE] [-i INF PER PAGE] [--[no-]automod] [-f FILTER]', + 'description': 'Grab infractions of a user, -f uses regex', + 'rich_description': 'Supports negative indexing in pager, flags are unix like', + 'permission': 'moderator', + 'execute': search_infractions_by_user + }, + 'get-infraction': { + 'alias': 'infraction-details' + }, + 'grab-infraction': { + 'alias': 'infraction-details' + }, + 'infraction-details': { + 'pretty_name': 'infraction-details ', + 'description': 'Grab details of an infractionID', + 'permission': 'moderator', + 'execute': get_detailed_infraction + }, + 'remove-infraction': { + 'alias': 'delete-infraction' + }, + 'rm-infraction': { + 'alias': 'delete-infraction' + }, + 'delete-infraction': { + 'pretty_name': 'delete-infraction ', + 'description': 'Delete an infraction by infractionID', + 'permission': 'administrator', + 'execute': delete_infraction + }, + } + +version_info: str = "1.2.13-DEV" diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index 8d0e8c8..d71d4e7 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -3,7 +3,7 @@ import importlib -import discord, time, asyncio, math, io, shlex, json +import discord, asyncio, json from dataclasses import dataclass import lib_db_obfuscator @@ -30,29 +30,19 @@ import lib_sonnetcommands importlib.reload(lib_sonnetcommands) -import lib_tparse - -importlib.reload(lib_tparse) from lib_goparsers import MustParseDuration -from lib_loaders import generate_infractionid, load_embed_color, embed_colors, datetime_now, datetime_unix +from lib_loaders import generate_infractionid, load_embed_color, embed_colors, datetime_now from lib_db_obfuscator import db_hlapi -from lib_parsers import parse_user_member, format_duration, paginate_noexcept +from lib_parsers import parse_user_member, format_duration from lib_compatibility import user_avatar_url -from lib_sonnetconfig import BOT_NAME, REGEX_VERSION +from lib_sonnetconfig import BOT_NAME from lib_sonnetcommands import CommandCtx -from lib_tparse import Parser import lib_constants as constants from typing import List, Tuple, Awaitable, Optional, Callable, Union, Final, Dict, cast import lib_lexdpyk_h as lexdpyk -# Import re to trick type checker into using re stubs -import re - -# Import into globals hashmap to ignore pyflakes redefinition errors -globals()["re"] = importlib.import_module(REGEX_VERSION) - # Catches error if the bot cannot message the user async def catch_dm_error(user: Union[discord.User, discord.Member], contents: discord.Embed, log_channel: Optional[discord.TextChannel]) -> None: @@ -575,180 +565,6 @@ async def unmute_user(message: discord.Message, args: List[str], client: discord return 0 -def get_user_id(s: str) -> int: - return int(s.strip("<@!>")) - - -async def search_infractions_by_user(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - tstart = time.monotonic() - - if not ctx.verbose: - raise lib_sonnetcommands.CommandError("ERROR: search-infractions only meant to be called directly") - - # Reparse args - try: - args = shlex.split(" ".join(args)) - except ValueError: - raise lib_sonnetcommands.CommandError("ERROR: Shlex failed to parse arguments") - - # Parse flags - parser = Parser("search-infractions") - - selected_chunk_f = parser.add_arg(["-p", "--page"], lambda s: int(s) - 1) - responsible_mod_f = parser.add_arg(["-m", "--mod"], get_user_id) - user_affected_f = parser.add_arg(["-u", "--user"], get_user_id) - per_page_f = parser.add_arg(["-i", "--infractioncount"], int) - infraction_type_f = parser.add_arg(["-t", "--type"], str) - filtering_f = parser.add_arg(["-f", "--filter"], str) - automod_f = lib_tparse.add_true_false_flag(parser, "automod") - - try: - parser.parse(args, stderr=io.StringIO(), exit_on_fail=False, lazy=True) - except lib_tparse.ParseFailureError: - await message.channel.send("Failed to parse flags") - return 1 - - selected_chunk = selected_chunk_f.get(0) - per_page = per_page_f.get(20) - user_affected = user_affected_f.get() - - # Default to user if no user/mod flags are supplied - if None is responsible_mod_f.get() is user_affected: - try: - user_affected = get_user_id(args[0]) - except (IndexError, ValueError): - pass - - if not 1 <= per_page <= 40: # pytype: disable=unsupported-operands - await message.channel.send("ERROR: Cannot exceed range 1-40 infractions per page") - return 1 - - refilter: "Optional[re.Pattern[str]]" - - if (f := filtering_f.get()) is not None: - try: - refilter = re.compile(f) - except re.error: - raise lib_sonnetcommands.CommandError("ERROR: Filter regex is invalid") - else: - refilter = None - - with db_hlapi(message.guild.id) as db: - if user_affected or responsible_mod_f.get(): - infractions = db.grab_filter_infractions(user=user_affected, moderator=responsible_mod_f.get(), itype=infraction_type_f.get(), automod=automod_f.get()) - assert isinstance(infractions, list) - else: - await message.channel.send("Please specify a user or moderator") - return 1 - - if refilter is not None: - infractions = [i for i in infractions if refilter.findall(i[4])] - - # Sort newest first - infractions.sort(reverse=True, key=lambda a: a[5]) - - # Return if no infractions, this is not an error as it returned a valid status - if not infractions: - await message.channel.send("No infractions found") - return 0 - - cpagecount = math.ceil(len(infractions) / per_page) - - # Test if valid page - if selected_chunk == -1: # ik it says page 0 but it does -1 on user input so the user would have entered 0 - raise lib_sonnetcommands.CommandError("ERROR: Cannot go to page 0") - elif selected_chunk < -1: - selected_chunk = (cpagecount + selected_chunk) + 1 - - def format_infraction(i: Tuple[str, str, str, str, str, int]) -> str: - return ', '.join([i[0], i[3], i[4]]) - - page = paginate_noexcept(infractions, selected_chunk, per_page, 1900, fmtfunc=format_infraction) - - tprint = (time.monotonic() - tstart) * 1000 - - await message.channel.send(f"Page {selected_chunk+1} / {cpagecount} ({len(infractions)} infraction{'s'*(len(infractions)!=1)}) ({tprint:.1f}ms)\n```css\nID, Type, Reason\n{page}```") - return 0 - - -async def get_detailed_infraction(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - if args: - with db_hlapi(message.guild.id) as db: - infraction = db.grab_infraction(args[0]) - if not infraction: - await message.channel.send("ERROR: Infraction ID does not exist") - return 1 - else: - await message.channel.send("ERROR: No argument supplied") - return 1 - - # Unpack this nightmare lmao - # pylint: disable=E0633 - infraction_id, user_id, moderator_id, infraction_type, reason, timestamp = infraction - - infraction_embed = discord.Embed(title="Infraction Search", description=f"Infraction for <@{user_id}>:", color=load_embed_color(message.guild, embed_colors.primary, ctx.ramfs)) - infraction_embed.add_field(name="Infraction ID", value=infraction_id) - infraction_embed.add_field(name="Moderator", value=f"<@{moderator_id}>") - infraction_embed.add_field(name="Type", value=infraction_type) - infraction_embed.add_field(name="Reason", value=reason) - - infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") - infraction_embed.timestamp = datetime_unix(int(timestamp)) - - try: - await message.channel.send(embed=infraction_embed) - return 0 - except discord.errors.Forbidden: - await message.channel.send(constants.sonnet.error_embed) - return 1 - - -async def delete_infraction(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - if args: - with db_hlapi(message.guild.id) as db: - infraction = db.grab_infraction(args[0]) - if not infraction: - await message.channel.send("ERROR: Infraction ID does not exist") - return 1 - # pylint: disable=E1136 - db.delete_infraction(infraction[0]) - else: - await message.channel.send("ERROR: No argument supplied") - return 1 - - if not ctx.verbose: - return 0 - - # pylint: disable=E0633 - infraction_id, user_id, moderator_id, infraction_type, reason, timestamp = infraction - - infraction_embed = discord.Embed(title="Infraction Deleted", description=f"Infraction for <@{user_id}>:", color=load_embed_color(message.guild, embed_colors.deletion, ctx.ramfs)) - infraction_embed.add_field(name="Infraction ID", value=infraction_id) - infraction_embed.add_field(name="Moderator", value=f"<@{moderator_id}>") - infraction_embed.add_field(name="Type", value=infraction_type) - infraction_embed.add_field(name="Reason", value=reason) - - infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") - - infraction_embed.timestamp = datetime_unix(int(timestamp)) - - try: - await message.channel.send(embed=infraction_embed) - return 0 - except discord.errors.Forbidden: - await message.channel.send(constants.sonnet.error_embed) - return 1 - - class purger: __slots__ = "user_id", @@ -798,85 +614,9 @@ async def purge_cli(message: discord.Message, args: List[str], client: discord.C return 1 -def notneg(v: int) -> int: - if v < 0: raise ValueError - return v - - -async def query_mutedb(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - parser = Parser("query_mutedb") - pageP = parser.add_arg(["-p", "--page"], lambda s: notneg(int(s) - 1)) - - try: - parser.parse(args, stderr=io.StringIO(), exit_on_fail=False, lazy=True) - except lib_tparse.ParseFailureError: - raise lib_sonnetcommands.CommandError("Failed to parse page") - - page = pageP.get(0) - - per_page = 10 - - with db_hlapi(message.guild.id) as db: - table: List[Tuple[str, str, int]] = db.fetch_guild_mutes() - - if not table: - await message.channel.send("No Muted users in database") - return 0 - - def fmtfunc(v: Tuple[str, str, int]) -> str: - ts = "No Unmute" if v[2] == 0 else format_duration(v[2] - datetime_now().timestamp()) - return (f"{v[1]}, {v[0]}, {ts}") - - out = paginate_noexcept(sorted(table, key=lambda i: i[2]), page, per_page, 1500, fmtfunc) - - await message.channel.send(f"Page {page+1} / {len(table)//per_page+1}, ({len(table)} mute{'s'*(len(table)!=1)})```css\nUid, InfractionID, Unmuted in\n{out}```") - return 0 - - -async def remove_mutedb(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: - if not message.guild: - return 1 - - try: - uid = int(args[0].strip("<@!>")) - except ValueError: - await message.channel.send("ERROR: Invalid user") - return 1 - except IndexError: - await message.channel.send("ERROR: No user specified") - return 1 - - with db_hlapi(message.guild.id) as db: - if db.is_muted(userid=uid): - db.unmute_user(userid=uid) - - await message.channel.send("Removed user from mute database") - - else: - await message.channel.send("ERROR: User is not in mute database") - return 1 - - return 0 - - category_info = {'name': 'moderation', 'pretty_name': 'Moderation', 'description': 'Moderation commands.'} commands = { - 'remove-mute': { - 'pretty_name': 'remove-mute ', - 'description': 'Removes a user from the mute database. Does not unmute in guild', - 'permission': 'administrator', - 'execute': remove_mutedb, - }, - 'list-mutes': { - 'pretty_name': 'list-mutes [-p PAGE]', - 'description': 'List all mutes in the mute database', - 'permission': 'moderator', - 'execute': query_mutedb, - }, 'warn': { 'pretty_name': 'warn [+modifiers] [reason]', 'description': 'Warn a user', @@ -926,47 +666,6 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco 'permission': 'moderator', 'execute': unmute_user }, - 'warnings': { - 'alias': 'search-infractions' - }, - 'list-infractions': { - 'alias': 'search-infractions' - }, - 'infractions': { - 'alias': 'search-infractions' - }, - 'search-infractions': - { - 'pretty_name': 'search-infractions <-u USER | -m MOD> [-t TYPE] [-p PAGE] [-i INF PER PAGE] [--[no-]automod] [-f FILTER]', - 'description': 'Grab infractions of a user, -f uses regex', - 'rich_description': 'Supports negative indexing in pager, flags are unix like', - 'permission': 'moderator', - 'execute': search_infractions_by_user - }, - 'get-infraction': { - 'alias': 'infraction-details' - }, - 'grab-infraction': { - 'alias': 'infraction-details' - }, - 'infraction-details': { - 'pretty_name': 'infraction-details ', - 'description': 'Grab details of an infractionID', - 'permission': 'moderator', - 'execute': get_detailed_infraction - }, - 'remove-infraction': { - 'alias': 'delete-infraction' - }, - 'rm-infraction': { - 'alias': 'delete-infraction' - }, - 'delete-infraction': { - 'pretty_name': 'delete-infraction ', - 'description': 'Delete an infraction by infractionID', - 'permission': 'administrator', - 'execute': delete_infraction - }, 'purge': { 'pretty_name': 'purge [user]', From 2e22a62c7dfee22e6408c2df1e78a646fa50d6af Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sun, 20 Mar 2022 23:36:23 -0700 Subject: [PATCH 19/78] add module version to category help --- cmds/cmd_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 0b95a9a..1069507 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -306,6 +306,8 @@ async def help_function(message: discord.Message, args: List[str], client: disco for name, desc in commands[page * per_page:(page * per_page) + per_page]: cmd_embed.add_field(name=name, value=desc, inline=False) + cmd_embed.set_footer(text=f"Module Version: {curmod.version_info}") + try: await message.channel.send(embed=cmd_embed) except discord.errors.Forbidden: From e92438c7392cc9bdd9ac2c7c7230537607809813 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 24 Mar 2022 14:49:52 -0700 Subject: [PATCH 20/78] add map-expand to help debug map exprs --- cmds/cmd_scripting.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 4769e3d..15a6505 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -5,7 +5,7 @@ import importlib -import shlex, discord, time, asyncio +import shlex, discord, time, asyncio, io import copy as pycopy import lib_parsers @@ -287,6 +287,38 @@ async def sonnet_async_map(message: discord.Message, args: List[str], client: di if ctx.verbose: await message.channel.send(f"Completed execution of {len(targs)} instances of {command} in {fmttime}ms") +async def sonnet_map_expansion(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: + if not message.guild: + return 1 + + try: + targs, _, command, exargs = await map_preprocessor_someexcept(message, args, client, ctx.cmds_dict, ctx.conf_cache, "map-expand") + except MapProcessError: + return 1 + + out = io.StringIO() + + cheader = ctx.conf_cache["prefix"] + command + + for i in targs: + + arguments = exargs[0] + i.split() + exargs[1] + + out.write(f'{cheader} {" ".join(arguments)}\n') + + data = out.getvalue() + + if len(data) <= 2000: + await message.channel.send(f"Expression expands to:\n```\n{data}```") + + else: + fp = discord.File(io.BytesIO(data.encode("utf8")), filename="map-expand.txt") + + await message.channel.send("Expansion too large to preview, sent as file:", files=[fp]) + + return 0 + + async def run_as_subcommand(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: # command check, perm check, run @@ -369,6 +401,13 @@ async def sleep_for(message: discord.Message, args: List[str], client: discord.C 'cache': 'keep', 'execute': sonnet_async_map }, + 'map-expand': + { + 'pretty_name': 'map-expand [-s args] [-e args] ()+', + 'description': 'Show what a map expression will expand to without actually running it', + 'permission': 'moderator', + 'execute': sonnet_map_expansion, + }, 'sub': { 'pretty_name': 'sub [args]+', 'description': 'runs a command as a subcommand', From d931b1df0d5eb7b4388fd42e65706abb4bd6735a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 24 Mar 2022 14:59:36 -0700 Subject: [PATCH 21/78] make sub message content change accordingly --- cmds/cmd_scripting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 15a6505..942458a 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -340,8 +340,10 @@ async def run_as_subcommand(message: discord.Message, args: List[str], client: d # set to subcommand newctx = pycopy.copy(ctx) newctx.verbose = False + newmsg = pycopy.copy(message) + newmsg.content = ctx.conf_cache["prefix"] + " ".join(args) - return await sonnetc.execute_ctx(message, args[1:], client, newctx) + return await sonnetc.execute_ctx(newmsg, args[1:], client, newctx) else: raise lib_sonnetcommands.CommandError("ERROR(sub): No command specified") From 4b324aad2a2cfd8f09d4f868f73512ade3118b84 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 26 Mar 2022 01:06:23 -0700 Subject: [PATCH 22/78] add total expansion limit to map commands --- cmds/cmd_scripting.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index 11680f5..c2500f8 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -173,6 +173,13 @@ async def map_preprocessor_someexcept(message: discord.Message, args: List[str], if command not in cmds_dict: raise lib_sonnetcommands.CommandError(f"ERROR({cname}): Command not found") + # get total length of -s and -e arguments multiplied by iteration count, projected memory use + memory_size = sum(len(item) for arglist in exargs for item in arglist) * len(targs) + + # disallow really large expansions + if memory_size >= 1 << 16: + raise lib_sonnetcommands.CommandError(f"ERROR({cname}): Total expansion size of arguments exceeds 64kb (projected at least {memory_size//1024} kbytes)") + cmd = SonnetCommand(cmds_dict[command], cmds_dict) if not await parse_permissions(message, conf_cache, cmd.permission): From b056727e5fd31a4786cd712d49394c81c0c12bed Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 28 Mar 2022 00:16:36 -0700 Subject: [PATCH 23/78] lib changes in preparation for pyright support --- libs/lib_loaders.py | 8 ++++---- libs/lib_starboard.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/lib_loaders.py b/libs/lib_loaders.py index b68e8bc..12677c9 100644 --- a/libs/lib_loaders.py +++ b/libs/lib_loaders.py @@ -291,10 +291,10 @@ def inc_statistics(indata: list[Any]) -> None: # I hate bugs more than I hate slow python class embed_colors: __slots__ = () - primary: Final = "primary" - creation: Final = "creation" - edit: Final = "edit" - deletion: Final = "deletion" + primary: Final[Literal["primary"]] = "primary" + creation: Final[Literal["creation"]] = "creation" + edit: Final[Literal["edit"]] = "edit" + deletion: Final[Literal["deletion"]] = "deletion" def load_embed_color(guild: discord.Guild, colortype: Literal["primary", "creation", "edit", "deletion"], ramfs: lexdpyk.ram_filesystem) -> int: diff --git a/libs/lib_starboard.py b/libs/lib_starboard.py index 0625ec1..e74f207 100644 --- a/libs/lib_starboard.py +++ b/libs/lib_starboard.py @@ -30,7 +30,7 @@ globals()["re"] = importlib.import_module(REGEX_VERSION) _image_filetypes = [".png", ".bmp", ".jpg", ".jpeg", ".gif", ".webp"] -_url_chars = "[a-zA-Z0-9\.\-_]" +_url_chars = r"[a-zA-Z0-9\.\-_]" # match https?:// # match optional newline From b7990e60dd8c444063d144b69b83695e766c4693 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 28 Mar 2022 01:23:01 -0700 Subject: [PATCH 24/78] update automod module for pyright --- cmds/cmd_automod.py | 123 +++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 64 deletions(-) diff --git a/cmds/cmd_automod.py b/cmds/cmd_automod.py index d438e49..1ac34a3 100644 --- a/cmds/cmd_automod.py +++ b/cmds/cmd_automod.py @@ -19,13 +19,17 @@ import lib_goparsers importlib.reload(lib_goparsers) +import lib_sonnetcommands + +importlib.reload(lib_sonnetcommands) from lib_goparsers import MustParseDuration from lib_db_obfuscator import db_hlapi from lib_sonnetconfig import REGEX_VERSION from lib_parsers import parse_role, parse_boolean, parse_user_member, format_duration +from lib_sonnetcommands import CommandCtx -from typing import Any, Dict, List, Callable, Coroutine, Tuple, Optional +from typing import Any, Dict, List, Callable, Coroutine, Tuple, Optional, Literal from typing import Final # pytype: disable=import-error import lib_constants as constants @@ -74,36 +78,34 @@ async def update_csv_blacklist(message: discord.Message, args: List[str], name: if verbose: await message.channel.send(f"Updated {name} successfully") -async def wb_change(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def wb_change(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await update_csv_blacklist(message, args, "word-blacklist", verbose=kwargs["verbose"], allowed=wb_allowedrunes) + await update_csv_blacklist(message, args, "word-blacklist", verbose=ctx.verbose, allowed=wb_allowedrunes) except blacklist_input_error: return 1 -async def word_in_word_change(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def word_in_word_change(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await update_csv_blacklist(message, args, "word-in-word-blacklist", verbose=kwargs["verbose"], allowed=wb_allowedrunes) + await update_csv_blacklist(message, args, "word-in-word-blacklist", verbose=ctx.verbose, allowed=wb_allowedrunes) except blacklist_input_error: return 1 -async def ftb_change(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def ftb_change(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await update_csv_blacklist(message, args, "filetype-blacklist", verbose=kwargs["verbose"]) + await update_csv_blacklist(message, args, "filetype-blacklist", verbose=ctx.verbose) except blacklist_input_error: return 1 -async def urlblacklist_change(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: - - verbose: bool = kwargs["verbose"] +async def urlblacklist_change(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await update_csv_blacklist(message, args, "url-blacklist", verbose=verbose, allowed=urlb_allowedrunes) + await update_csv_blacklist(message, args, "url-blacklist", verbose=ctx.verbose, allowed=urlb_allowedrunes) except blacklist_input_error: return 1 @@ -190,37 +192,37 @@ async def remove_regex_type(message: discord.Message, args: List[str], db_entry: if verbose: await message.channel.send("Successfully Updated RegEx") -async def regexblacklist_add(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def regexblacklist_add(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await add_regex_type(message, args, "regex-blacklist", verbose=kwargs["verbose"]) + await add_regex_type(message, args, "regex-blacklist", verbose=ctx.verbose) except blacklist_input_error: return 1 -async def regexblacklist_remove(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def regexblacklist_remove(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await remove_regex_type(message, args, "regex-blacklist", verbose=kwargs["verbose"]) + await remove_regex_type(message, args, "regex-blacklist", verbose=ctx.verbose) except blacklist_input_error: return 1 -async def regex_notifier_add(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def regex_notifier_add(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await add_regex_type(message, args, "regex-notifier", verbose=kwargs["verbose"]) + await add_regex_type(message, args, "regex-notifier", verbose=ctx.verbose) except blacklist_input_error: return 1 -async def regex_notifier_remove(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def regex_notifier_remove(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: - await remove_regex_type(message, args, "regex-notifier", verbose=kwargs["verbose"]) + await remove_regex_type(message, args, "regex-notifier", verbose=ctx.verbose) except blacklist_input_error: return 1 -async def list_blacklist(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def list_blacklist(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: - mconf: Dict[str, Any] = kwargs["conf_cache"] + mconf: Dict[str, Any] = ctx.conf_cache raw = False if args and args[0] in ["--raw", "-r"]: @@ -262,14 +264,14 @@ async def list_blacklist(message: discord.Message, args: List[str], client: disc await message.channel.send(errmsg, file=fileobj) -async def set_blacklist_infraction_level(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def set_blacklist_infraction_level(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: if not message.guild: return 1 if args: action = args[0].lower() else: - await message.channel.send(f"blacklist action is `{kwargs['conf_cache']['blacklist-action']}`") + await message.channel.send(f"blacklist action is `{ctx.conf_cache['blacklist-action']}`") return if not action in ["warn", "kick", "mute", "ban"]: @@ -279,68 +281,59 @@ async def set_blacklist_infraction_level(message: discord.Message, args: List[st with db_hlapi(message.guild.id) as database: database.add_config("blacklist-action", action) - if kwargs["verbose"]: await message.channel.send(f"Updated blacklist action to `{action}`") + if ctx.verbose: await message.channel.send(f"Updated blacklist action to `{action}`") -async def change_rolewhitelist(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def change_rolewhitelist(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: - return await parse_role(message, args, "blacklist-whitelist", verbose=kwargs["verbose"]) + return await parse_role(message, args, "blacklist-whitelist", verbose=ctx.verbose) -async def antispam_set(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def antispam_set(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: if not message.guild: return 1 - if not args: - antispam = kwargs["conf_cache"]["antispam"] - await message.channel.send(f"Antispam timings are M:{antispam[0]},S:{antispam[1]}") - return - if len(args) == 1: try: messages, seconds = [float(i) for i in args[0].split(",")] except ValueError: - await message.channel.send("ERROR: Incorrect args supplied") - return 1 + raise lib_sonnetcommands.CommandError("ERROR: Incorrect args supplied") elif len(args) > 1: try: messages = float(args[0]) seconds = float(args[1]) except ValueError: - await message.channel.send("ERROR: Incorrect args supplied") - return 1 + raise lib_sonnetcommands.CommandError("ERROR: Incorrect args supplied") + + else: + antispam = ctx.conf_cache["antispam"] + await message.channel.send(f"Antispam timings are M:{antispam[0]},S:{antispam[1]}") + return 0 # Prevent bullshit outside_range = "ERROR: Cannot go outside range" if not (2 <= messages <= 64): - await message.channel.send(f"{outside_range} 2-64 messages") - return 1 + raise lib_sonnetcommands.CommandError(f"{outside_range} 2-64 messages") elif not (0 <= seconds <= 10): - await message.channel.send(f"{outside_range} 0-10 seconds") - return 1 + raise lib_sonnetcommands.CommandError(f"{outside_range} 0-10 seconds") with db_hlapi(message.guild.id) as database: database.add_config("antispam", f"{int(messages)},{seconds}") - if kwargs["verbose"]: await message.channel.send(f"Updated antispam timings to M:{int(messages)},S:{seconds}") + if ctx.verbose: await message.channel.send(f"Updated antispam timings to M:{int(messages)},S:{seconds}") + return 0 -async def char_antispam_set(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def char_antispam_set(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Optional[Literal[0, 1]]: if not message.guild: return 1 - if not args: - antispam = kwargs["conf_cache"]["char-antispam"] - await message.channel.send(f"CharAntispam timings are M:{antispam[0]},S:{antispam[1]},C:{antispam[2]}") - return - if len(args) == 1: try: messages, seconds, chars = [float(i) for i in args[0].split(",")] except ValueError: - await message.channel.send("ERROR: Incorrect args supplied") - return 1 + raise lib_sonnetcommands.CommandError("ERROR: Incorrect args supplied") elif len(args) > 1: try: @@ -348,28 +341,30 @@ async def char_antispam_set(message: discord.Message, args: List[str], client: d seconds = float(args[1]) chars = float(args[2]) except ValueError: - await message.channel.send("ERROR: Incorrect args supplied") - return 1 + raise lib_sonnetcommands.CommandError("ERROR: Incorrect args supplied") + + else: + antispam = ctx.conf_cache["char-antispam"] + await message.channel.send(f"CharAntispam timings are M:{antispam[0]},S:{antispam[1]},C:{antispam[2]}") + return 0 # Prevent bullshit outside_range = "ERROR: Cannot go outside range" if not (2 <= messages <= 64): - await message.channel.send(f"{outside_range} 2-64 messages") - return 1 + raise lib_sonnetcommands.CommandError(f"{outside_range} 2-64 messages") elif not (0 <= seconds <= 10): - await message.channel.send(f"{outside_range} 0-10 seconds") - return 1 + raise lib_sonnetcommands.CommandError(f"{outside_range} 0-10 seconds") elif not (128 <= chars <= 2**16): - await message.channel.send(f"{outside_range} 128-{2**16} chars") - return 1 + raise lib_sonnetcommands.CommandError(f"{outside_range} 128-{2**16} chars") with db_hlapi(message.guild.id) as database: database.add_config("char-antispam", f"{int(messages)},{seconds},{int(chars)}") - if kwargs["verbose"]: await message.channel.send(f"Updated char antispam timings to M:{int(messages)},S:{seconds},C:{int(chars)}") + if ctx.verbose: await message.channel.send(f"Updated char antispam timings to M:{int(messages)},S:{seconds},C:{int(chars)}") + return 0 -async def antispam_time_set(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def antispam_time_set(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: if not message.guild: return 1 @@ -380,7 +375,7 @@ async def antispam_time_set(message: discord.Message, args: List[str], client: d await message.channel.send("ERROR: Invalid time format") return 1 else: - mutetime = int(kwargs["conf_cache"]["antispam-time"]) + mutetime = int(ctx.conf_cache["antispam-time"]) await message.channel.send(f"Antispam mute time is {mutetime} seconds") return 0 @@ -395,7 +390,7 @@ async def antispam_time_set(message: discord.Message, args: List[str], client: d with db_hlapi(message.guild.id) as db: db.add_config("antispam-time", str(mutetime)) - if kwargs["verbose"]: await message.channel.send(f"Set antispam mute time to {format_duration(mutetime)}") + if ctx.verbose: await message.channel.send(f"Set antispam mute time to {format_duration(mutetime)}") class NoGuildError(Exception): @@ -410,7 +405,7 @@ def __init__(self, message: discord.Message): raise NoGuildError(f"{message}: contains no guild") self.m: Final[discord.Message] = message - self.guild = message.guild + self.guild: Final = message.guild self.ops: Dict[str, Tuple[Callable[[List[str], discord.Client], Coroutine[Any, Any, int]], str]] = { # pytype: disable=annotation-type-mismatch "user": (self.useredit, "add|remove 'Add or remove a userid from the watchlist'"), @@ -536,7 +531,7 @@ async def defaultpfpedit(self, args: List[str], client: discord.Client) -> int: return 0 -async def add_joinrule(message: discord.Message, args: List[str], client: discord.Client, **kwargs: Any) -> Any: +async def add_joinrule(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> Any: try: rules = joinrules(message) @@ -695,4 +690,4 @@ async def add_joinrule(message: discord.Message, args: List[str], client: discor }, } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" From efa0ea811fc1827fad78549fd6ac04cfee80b61a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:22:24 -0700 Subject: [PATCH 25/78] update more libraries for pyright --- libs/lib_goparsers.py | 20 +++++++++++--------- libs/lib_sonnetconfig.py | 4 ++++ libs/lib_sonnetdb.py | 8 +++++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 5490f8d..a3e4545 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -15,7 +15,7 @@ import ctypes as _ctypes import subprocess as _subprocess -from typing import cast +from typing import cast, Optional import lib_sonnetconfig @@ -34,8 +34,8 @@ class _ParseDurationRet(_ctypes.Structure): _fields_ = [("ret", _ctypes.c_longlong), ("err", _ctypes.c_int)] -hascompiled = True _version = "2.0.0-DEV.3" +_gotools: Optional[_ctypes.CDLL] if GOLIB_LOAD: try: _gotools = _ctypes.CDLL(f"./libs/compiled/gotools.{_version}.so") @@ -44,13 +44,15 @@ class _ParseDurationRet(_ctypes.Structure): if _subprocess.run(["make", "gotools", f"GOCMD={GOLIB_VERSION}"]).returncode == 0: _gotools = _ctypes.CDLL(f"./libs/compiled/gotools.{_version}.so") else: - hascompiled = False + _gotools = None except OSError: - hascompiled = False + _gotools = None else: - hascompiled = False + _gotools = None -if hascompiled: +hascompiled = _gotools is not None + +if _gotools is not None: _gotools.ParseDuration.argtypes = [_GoString] _gotools.ParseDuration.restype = _ParseDurationRet _gotools.GenerateCacheFile.argtypes = [_GoString, _GoString] @@ -106,7 +108,7 @@ def GenerateCacheFile(fin: str, fout: str) -> None: :raises: FileNotFoundError - infile does not exist """ - if hascompiled: + if _gotools is not None: ret = _gotools.GenerateCacheFile(_FromString(fin), _FromString(fout)) @@ -144,7 +146,7 @@ def ParseDuration(s: str) -> int: :returns: int - Time parsed in seconds """ - if not hascompiled: + if _gotools is None: raise errors.NoBinaryError("ParseDuration: No binary found") # Special case to default to seconds @@ -172,7 +174,7 @@ def MustParseDuration(s: str) -> int: :returns: int - Time parsed in seconds """ - if hascompiled: + if _gotools is not None: return ParseDuration(s) diff --git a/libs/lib_sonnetconfig.py b/libs/lib_sonnetconfig.py index 379519c..b9279fe 100644 --- a/libs/lib_sonnetconfig.py +++ b/libs/lib_sonnetconfig.py @@ -33,12 +33,16 @@ def _load_cfg(attr: str, default: Typ, typ: Type[Typ], testfunc: Optional[Callab conf: Union[Any, Typ] = getattr(sonnet_cfg, attr, default) + # Asserts that conf is of type Typ if false if not isinstance(conf, typ): raise TypeError(f"Sonnet Config {attr} is not type {typ.__name__}") if testfunc is not None and not testfunc(conf): raise TypeError(f"Sonnet Config {attr}: {errmsg}") + # pyright thinks that it can still be Any despite isinstance check + # This applies to the whole file but mypy interferes with type: ignore syntax ;-; + # pyright: reportGeneralTypeIssues=false return conf diff --git a/libs/lib_sonnetdb.py b/libs/lib_sonnetdb.py index 74dc2df..b9492a4 100644 --- a/libs/lib_sonnetdb.py +++ b/libs/lib_sonnetdb.py @@ -22,7 +22,7 @@ import json from lib_mdb_handler import db_handler, db_error with open(".login-info.txt", encoding="utf-8") as login_info_file: # Grab login data - db_connection_parameters = json.load(login_info_file) + db_connection_parameters: Any = json.load(login_info_file) elif DB_TYPE == "sqlite3": import lib_sql_handler @@ -30,6 +30,9 @@ from lib_sql_handler import db_handler, db_error # type: ignore[misc] db_connection_parameters = SQLITE3_LOCATION +else: + raise RuntimeError("Could not load database backend (non valid specifier)") + class DATABASE_FATAL_CONNECTION_LOSS(Exception): __slots__ = () @@ -125,6 +128,9 @@ def inject_enum(self, enumname: str, schema: List[Tuple[str, Type[Union[str, int pks: Any = (PK, tuple, 1) elif T == int: pks = (PK, int(64), 1) + else: + raise TypeError("Invalid schema passed") + cols: List[Any] = [pks] # Inject rest of table From 7f8c21542b810036bb1541da2493224421fbc17d Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:25:02 -0700 Subject: [PATCH 26/78] update administration for pyright --- cmds/cmd_administration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_administration.py b/cmds/cmd_administration.py index 1216647..93bab53 100644 --- a/cmds/cmd_administration.py +++ b/cmds/cmd_administration.py @@ -277,6 +277,7 @@ async def gdpr_database(message: discord.Message, args: List[str], client: disco confirmation = "" else: command = "" + confirmation = "" PREFIX = ctx.conf_cache["prefix"] @@ -509,4 +510,4 @@ async def set_filelog_behavior(message: discord.Message, args: List[str], client } } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" From 3aa515c1c3c163e118e16ca72accbf3b5d625fb1 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:29:48 -0700 Subject: [PATCH 27/78] LeXdPyK 1.4.11: add pyright support --- main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index daea9e1..703a6a3 100644 --- a/main.py +++ b/main.py @@ -172,7 +172,9 @@ def read_f(self, dirstr: Optional[str] = None, dirlist: Optional[List[str]] = No def create_f(self, dirstr: Optional[str] = None, dirlist: Optional[List[str]] = None, f_type: Optional[Type[Any]] = None, f_args: Optional[List[Any]] = None) -> Any: - f_type = io.BytesIO if f_type is None else f_type + if f_type is None: + f_type = io.BytesIO + f_args = [] if f_args is None else f_args file_to_write = self._parsedirlist(dirstr, dirlist) @@ -899,7 +901,7 @@ def main(args: List[str]) -> int: # Define version info and start time -version_info: str = "LeXdPyK 1.4.10" +version_info: str = "LeXdPyK 1.4.11" bot_start_time: float = time.time() if __name__ == "__main__": From 1e374040b2320b529dd17d33cb5406183bae41ca Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:29:56 -0700 Subject: [PATCH 28/78] add pyright config file --- pyrightconfig.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 pyrightconfig.json diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..ea05489 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,5 @@ +{ + "extraPaths": ["libs/", "common/"], + "pythonVersion": "3.8", + "pythonPlatform": "Linux" +} From 4b9129b0203aec254aec2b099eb4fbc7421ab949 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:34:09 -0700 Subject: [PATCH 29/78] yapf pass --- libs/lib_sonnetdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/lib_sonnetdb.py b/libs/lib_sonnetdb.py index b9492a4..94cb2d8 100644 --- a/libs/lib_sonnetdb.py +++ b/libs/lib_sonnetdb.py @@ -131,7 +131,6 @@ def inject_enum(self, enumname: str, schema: List[Tuple[str, Type[Union[str, int else: raise TypeError("Invalid schema passed") - cols: List[Any] = [pks] # Inject rest of table for col in schema[1:]: From 0b99784cd056e4f4e1cd0f8bc7eceec385147dae Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:36:26 -0700 Subject: [PATCH 30/78] add pyright to gh actions --- .github/workflows/python-dev.yml | 5 ++++- .github/workflows/python-package.yml | 3 +++ build_tools/autotest.py | 1 + requirements.buildtime.txt | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a048566..4311393 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -26,7 +26,7 @@ jobs: cache: 'pip' - name: Install base deps run: | - python -m pip install pyflakes mypy + python -m pip install pyflakes mypy pyright python -m pip install -r requirements.txt - name: run unicode safety checker (ensures no malicious unicode codepoints) run: | @@ -49,3 +49,6 @@ jobs: - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable + - name: run pyright type checking + run: | + pyright diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d6a62fa..20ae977 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -50,6 +50,9 @@ jobs: - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable + - name: run pyright type checking + run: | + pyright - name: yapf integrity check run: | yapf -drp . diff --git a/build_tools/autotest.py b/build_tools/autotest.py index c2d0723..ad85451 100644 --- a/build_tools/autotest.py +++ b/build_tools/autotest.py @@ -74,6 +74,7 @@ def main() -> None: "mypy": "mypy . --ignore-missing-imports --strict --warn-unreachable --python-version 3.8", "yapf": "yapf -drp .", "pylint": Shell("pylint **/*.py -E -j4 --py-version=3.8"), + "pyright": "pyright", #"pytype": "pytype .", } diff --git a/requirements.buildtime.txt b/requirements.buildtime.txt index dc24790..b01b755 100644 --- a/requirements.buildtime.txt +++ b/requirements.buildtime.txt @@ -1,4 +1,4 @@ -pytype +pyright pylint yapf mypy From 5b570c1ab13de647c7790548c7a6ba867a1773c3 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:48:35 -0700 Subject: [PATCH 31/78] fix protocol errors --- libs/lib_loaders.py | 2 +- libs/lib_tparse.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/lib_loaders.py b/libs/lib_loaders.py index 12677c9..d629b17 100644 --- a/libs/lib_loaders.py +++ b/libs/lib_loaders.py @@ -91,7 +91,7 @@ def read(self, size: int = -1) -> bytes: class Writer(Protocol): - def write(self, data: bytes) -> int: + def write(self, data: bytes, /) -> int: ... diff --git a/libs/lib_tparse.py b/libs/lib_tparse.py index 17a4fce..71f4d16 100644 --- a/libs/lib_tparse.py +++ b/libs/lib_tparse.py @@ -28,7 +28,7 @@ class StringWriter(Protocol): """ A StringWriter is an interface for any object (T) containing a T.write(s: str) -> int method """ - def write(self, s: str) -> int: + def write(self, s: str, /) -> int: ... From 8d2ef947e7d0e49e58fb185d7d4600117a4a82e8 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:50:50 -0700 Subject: [PATCH 32/78] remove pyright from dev tests --- .github/workflows/python-dev.yml | 5 +---- .github/workflows/python-package.yml | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 4311393..a048566 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -26,7 +26,7 @@ jobs: cache: 'pip' - name: Install base deps run: | - python -m pip install pyflakes mypy pyright + python -m pip install pyflakes mypy python -m pip install -r requirements.txt - name: run unicode safety checker (ensures no malicious unicode codepoints) run: | @@ -49,6 +49,3 @@ jobs: - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable - - name: run pyright type checking - run: | - pyright diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 20ae977..eaaab79 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -50,9 +50,6 @@ jobs: - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable - - name: run pyright type checking - run: | - pyright - name: yapf integrity check run: | yapf -drp . @@ -61,6 +58,9 @@ jobs: sudo apt update sudo apt install libmariadb-dev-compat python -m pip install mariadb + - name: run pyright type checking + run: | + pyright - name: linting pass run: | pylint **/*.py -E -j4 From cb6b8a9809949163604f34108ba0b6acee2ba414 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 01:54:15 -0700 Subject: [PATCH 33/78] fix more pyright issues --- libs/lib_encryption_wrapper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/lib_encryption_wrapper.py b/libs/lib_encryption_wrapper.py index 1eb720a..94bdaaf 100644 --- a/libs/lib_encryption_wrapper.py +++ b/libs/lib_encryption_wrapper.py @@ -37,10 +37,10 @@ def directBinNumber(inData: int, length: int) -> bytes: class _WriteSeekCloser(Protocol): - def write(self, buf: bytes) -> int: + def write(self, buf: bytes, /) -> int: ... - def seek(self, cookie: int, whence: int = 0) -> int: + def seek(self, cookie: int, whence: int = 0, /) -> int: ... def close(self) -> None: @@ -132,7 +132,7 @@ class _ReadSeekCloser(Protocol): def read(self, amnt: int) -> bytes: ... - def seek(self, cookie: int, whence: int = 0) -> int: + def seek(self, cookie: int, whence: int = 0, /) -> int: ... def close(self) -> None: From a2b7e02febcea7e65f5e152760886d04eea3d126 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 21:47:40 -0700 Subject: [PATCH 34/78] add anti alias -> alias command map validation --- build_tools/cmds_to_html.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build_tools/cmds_to_html.py b/build_tools/cmds_to_html.py index 4a56c97..1db5e50 100644 --- a/build_tools/cmds_to_html.py +++ b/build_tools/cmds_to_html.py @@ -73,12 +73,14 @@ raise SyntaxError(f"ERROR IN [{execmodule} : {command}] PERMISSION TYPE({cmd.permission}) IS NOT VALID") -# Test for aliases pointing to existing commands +# Test for aliases pointing to existing commands that are not aliases for command in command_modules_dict: if "alias" not in command_modules_dict[command]: continue - if command_modules_dict[command]['alias'] in command_modules_dict: + if (cname := command_modules_dict[command]['alias']) in command_modules_dict: + if 'alias' in command_modules_dict[cname]: + raise SyntaxError(f"ERROR IN ALIAS:{command}, POINTING TOWARDS COMMAND THAT IS ALSO ALIAS: {cname}\n(EXPECTED NON ALIAS ENTRY)") continue raise SyntaxError(f"ERROR IN ALIAS:{command}, NO SUCH COMMAND {command_modules_dict[command]['alias']}") From 62eb973b6b6d1de1c39784b1d993492286f3a638 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 22:20:53 -0700 Subject: [PATCH 35/78] add process time to help command --- cmds/cmd_utils.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 1069507..5c4a55c 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -160,10 +160,37 @@ async def avatar_function(message: discord.Message, args: List[str], client: dis raise lib_sonnetcommands.CommandError(constants.sonnet.error_embed) +class Duration(int): + """ + A duration represented as nanoseconds with helper methods for conversions + """ + def micro(self) -> int: + return self // 1000 + + def milli(self) -> int: + return self // 1000 // 1000 + + def milli_f(self) -> float: + return self / 1000 / 1000 + + def sec(self) -> int: + return self // 1000 // 1000 // 1000 + + +# based on rusts std::time::Instant api +class Instant(int): + @staticmethod + def now() -> "Instant": + return Instant(time.monotonic_ns()) + + def elapsed(self) -> Duration: + return Duration(time.monotonic_ns() - self) + + class HelpHelper: - __slots__ = "guild", "args", "client", "ctx", "prefix", "helpname", "message" + __slots__ = "guild", "args", "client", "ctx", "prefix", "helpname", "message", "start_time" - def __init__(self, message: discord.Message, guild: discord.Guild, args: List[str], client: discord.Client, ctx: CommandCtx, prefix: str, helpname: str): + def __init__(self, message: discord.Message, guild: discord.Guild, args: List[str], client: discord.Client, ctx: CommandCtx, prefix: str, helpname: str, start_time: Instant): self.message = message self.guild = guild self.args = args @@ -171,6 +198,7 @@ def __init__(self, message: discord.Message, guild: discord.Guild, args: List[st self.ctx = ctx self.prefix = prefix self.helpname = helpname + self.start_time = start_time # Builds a single command async def single_command(self, cmd_name: str) -> discord.Embed: @@ -207,6 +235,8 @@ async def single_command(self, cmd_name: str) -> discord.Embed: if aliases: cmd_embed.add_field(name="Aliases:", value=aliases, inline=False) + cmd_embed.set_footer(text=f"Took: {self.start_time.elapsed().milli_f():.1f}ms") + return cmd_embed # Builds help for a category @@ -255,7 +285,7 @@ async def full_help(self, page: int, per_page: int) -> discord.Embed: helptext = ', '.join(sorted(mnames)) if mnames else module.category_info['description'] cmd_embed.add_field(name=f"{module.category_info['pretty_name']} ({module.category_info['name']})", value=helptext, inline=False) - cmd_embed.set_footer(text=f"Total Commands: {total} | Total Endpoints: {len(cmds_dict)}") + cmd_embed.set_footer(text=f"Total Commands: {total} | Total Endpoints: {len(cmds_dict)} | Took: {self.start_time.elapsed().milli_f():.1f}ms") return cmd_embed @@ -264,6 +294,8 @@ async def help_function(message: discord.Message, args: List[str], client: disco if not message.guild: return 1 + start_time = Instant.now() + helpname: str = f"{BOT_NAME} Help" per_page: int = 10 @@ -283,7 +315,7 @@ async def help_function(message: discord.Message, args: List[str], client: disco commandonly = commandonlyP.get() is True prefix = ctx.conf_cache["prefix"] - help_helper = HelpHelper(message, message.guild, args, client, ctx, prefix, helpname) + help_helper = HelpHelper(message, message.guild, args, client, ctx, prefix, helpname, start_time) if args: @@ -306,7 +338,7 @@ async def help_function(message: discord.Message, args: List[str], client: disco for name, desc in commands[page * per_page:(page * per_page) + per_page]: cmd_embed.add_field(name=name, value=desc, inline=False) - cmd_embed.set_footer(text=f"Module Version: {curmod.version_info}") + cmd_embed.set_footer(text=f"Module Version: {curmod.version_info} | Took {start_time.elapsed().milli_f():.1f}ms") try: await message.channel.send(embed=cmd_embed) From 70002826ae3061e08dd9f4d12ab481a9897b9557 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 30 Mar 2022 22:33:48 -0700 Subject: [PATCH 36/78] make single command help footer less barren --- cmds/cmd_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 5c4a55c..458b1a5 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -235,7 +235,9 @@ async def single_command(self, cmd_name: str) -> discord.Embed: if aliases: cmd_embed.add_field(name="Aliases:", value=aliases, inline=False) - cmd_embed.set_footer(text=f"Took: {self.start_time.elapsed().milli_f():.1f}ms") + module: lexdpyk.cmd_module = next(i for i in self.ctx.cmds if cmd_name in i.commands) + + cmd_embed.set_footer(text=f"Module: {module.category_info['pretty_name']} | Took: {self.start_time.elapsed().milli_f():.1f}ms") return cmd_embed From 7fb50e811a5bbf9f25571d4d6c252e3978fcd5b4 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 6 Apr 2022 22:06:30 -0400 Subject: [PATCH 37/78] make pyright ignore missing imports like mypy --- pyrightconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index ea05489..b4ab2ff 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,5 +1,6 @@ { "extraPaths": ["libs/", "common/"], "pythonVersion": "3.8", - "pythonPlatform": "Linux" + "pythonPlatform": "Linux", + "reportMissingImports": false } From a2d9e97252d0e7635124120510aa2e104899fce3 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 6 Apr 2022 22:07:08 -0400 Subject: [PATCH 38/78] make commands runnable using a mention as a prefix --- dlibs/dlib_messages.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dlibs/dlib_messages.py b/dlibs/dlib_messages.py index 10e2789..de3031d 100644 --- a/dlibs/dlib_messages.py +++ b/dlibs/dlib_messages.py @@ -483,6 +483,8 @@ async def on_message(message: discord.Message, kernel_args: lexdpyk.KernelArgs) if not message_deleted: asyncio.create_task(log_message_files(message, kernel_args.kernel_ramfs)) + mention_prefix: Final = message.content.startswith(f"<@{client.user.id}>") or message.content.startswith(f"<@!{client.user.id}>") + # Check if this is meant for us. if not (message.content.startswith(mconf["prefix"])) or message_deleted: if client.user.mentioned_in(message) and str(client.user.id) == message.content.strip("<@!>"): @@ -490,11 +492,21 @@ async def on_message(message: discord.Message, kernel_args: lexdpyk.KernelArgs) await message.channel.send(f"My prefix for this guild is {mconf['prefix']}") except discord.errors.Forbidden: pass # Nothing we can do if we lack perms to speak - return + return + elif not mention_prefix: + return # Split into cmds and arguments. arguments: Final = message.content.split() - command = arguments[0][len(mconf["prefix"]):] + if mention_prefix: + try: + # delete mention + del arguments[0] + command = arguments[0] + except IndexError: + return + else: + command = arguments[0][len(mconf["prefix"]):] # Remove command from the arguments. del arguments[0] From 2ff588636a746acccf5415303755365c85b338ff Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 14 Apr 2022 19:25:46 -0700 Subject: [PATCH 39/78] add extra permissions documented in discord dev docs --- libs/lib_constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/lib_constants.py b/libs/lib_constants.py index 9391b6e..55cea53 100644 --- a/libs/lib_constants.py +++ b/libs/lib_constants.py @@ -41,12 +41,14 @@ class permission: 29: "manage webhooks", 30: "manage emojis and stickers", 31: "use slash commands", + 32: "request to speak", # NOTE(ultrabear) this is listed as 'may be removed' 33: "manage events", 34: "manage threads", 35: "create public threads", 36: "create private threads", 37: "use external stickers", 38: "send messages in threads", + 39: "use activities", 40: "moderate members", } From 136b17bf35fae59fd3b49f820bb1c1380143dd61 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 14 Apr 2022 19:59:59 -0700 Subject: [PATCH 40/78] make roleinfo with no perms return special case None --- cmds/cmd_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 458b1a5..15b1af2 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -412,7 +412,10 @@ def perms_to_str(p: discord.Permissions) -> str: values.sort() - return f"`{'` `'.join(values)}`" + if values: + return f"`{'` `'.join(values)}`" + else: + return "None" async def grab_role_info(message: discord.Message, args: List[str], client: discord.Client, ctx: CommandCtx) -> int: From 0ffbe5b115a9f0e121bf2f04a6092920726344f9 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 20 Apr 2022 11:39:09 -0700 Subject: [PATCH 41/78] rewrite db_hlapi.fetch_all_mutes to remove a type ignore --- libs/lib_sonnetdb.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/lib_sonnetdb.py b/libs/lib_sonnetdb.py index 94cb2d8..2f33aa6 100644 --- a/libs/lib_sonnetdb.py +++ b/libs/lib_sonnetdb.py @@ -511,13 +511,17 @@ def fetch_all_mutes(self) -> List[Tuple[str, str, str, int]]: """ # Grab list of tables - tablelist = self._db.list_tables("%_mutes") + guild_list: Tuple[List[str], ...] = self._db.list_tables("%_mutes") - mutetable: List[Tuple[str, str, str, int]] = [] - for i in tablelist: - mutetable.extend([(i[0][:-6], ) + tuple(a) for a in self._db.fetch_table(i[0])]) # type: ignore[misc] + mute_table: List[Tuple[str, str, str, int]] = [] + for i in guild_list: + guild_id = str(i[0][:-6]) + for row in self._db.fetch_table(guild_id): + # assert types at runtime + infraction_id, user_id, unmute_time = str(row[0]), str(row[1]), int(row[2]) + mute_table.append((guild_id, infraction_id, user_id, unmute_time)) - return mutetable + return mute_table def is_muted(self, userid: Optional[int] = None, infractionid: Optional[str] = None) -> bool: """ From 3cf2fe7bdcf8211691c8dba137e9dc4bedf5ad05 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 15:51:38 -0700 Subject: [PATCH 42/78] add gotools.ParseDurationSuper, pure python duration parser TODO go variant --- build_tools/manualtest.py | 102 ++++++++++++++++++++++ libs/lib_goparsers.py | 178 +++++++++++++++++++++++++++++++++++--- 2 files changed, 266 insertions(+), 14 deletions(-) create mode 100644 build_tools/manualtest.py diff --git a/build_tools/manualtest.py b/build_tools/manualtest.py new file mode 100644 index 0000000..c8cdef1 --- /dev/null +++ b/build_tools/manualtest.py @@ -0,0 +1,102 @@ +import sys +import os + +sys.path.insert(1, os.getcwd() + '/common') +sys.path.insert(1, os.getcwd() + '/libs') + +from lib_goparsers import ParseDurationSuper + +from typing import Callable, TypeVar, List, Optional, Final, Iterable + +T = TypeVar("T") +O = TypeVar("O") + + +def test_func_io(func: Callable[[T], O], arg: T, expect: O) -> None: + assert func(arg) == expect, f"func({arg=})={func(arg)} != {expect=}" + + +def test_parse_duration() -> Optional[Iterable[Exception]]: + WEEK: Final = 7 * 24 * 60 * 60 + DAY: Final = 24 * 60 * 60 + HOUR: Final = 60 * 60 + MINUTE: Final = 60 + SECOND: Final = 1 + tests = ( + [ + # "Real user" tests, general correctness + ("123", 123), + ("5minutes", 5 * MINUTE), + ("45s", 45), + ("s", 1), + # Various rejection paths + ("5monite", None), + ("sfgdsgf", None), + ("minutes5", None), + ("5seconds4", None), + ("seconds5m", None), + ("", None), + ("josh", None), + ("seconds5seconds", None), + ("1w1wday", None), + ("1day2weeks7dam", None), + # Test all unit names have correct outputs + ("1w1week1weeks", 3 * WEEK), + ("1d1day1days", 3 * DAY), + ("1h1hour1hours", 3 * HOUR), + ("1m1minute1minutes", 3 * MINUTE), + ("1s1second1seconds", 3 * SECOND), + # Test all single unit cases + ("week", WEEK), + ("w", WEEK), + ("day", DAY), + ("d", DAY), + ("hour", HOUR), + ("h", HOUR), + ("minute", MINUTE), + ("m", MINUTE), + ("second", SECOND), + ("s", SECOND), + # Test for floating point accuracy + (f"{(1<<54)+1}m1s", ((1 << 54) + 1) * 60 + 1), + ("4.5h", 4 * HOUR + 30 * MINUTE), + ("4.7h", 4 * HOUR + (7 * HOUR // 10)), + ("3.5d7.3m", 3 * DAY + 12 * HOUR + 7 * MINUTE + (3 * MINUTE // 10)), + # Test for fp parse rejection + ("5.6.34seconds", None), + ] + ) + out = [] + + for i in tests: + try: + test_func_io(ParseDurationSuper, i[0], i[1]) + except AssertionError as e: + out.append(e) + + if out: return out + else: + return None + + +testfuncs: List[Callable[[], Optional[Iterable[Exception]]]] = [test_parse_duration] + + +def main_tests() -> None: + + failure = False + + for i in testfuncs: + errs = i() + if errs is not None: + failure = True + print(i) + for e in errs: + print(e) + + if failure: + sys.exit(1) + + +if __name__ == "__main__": + main_tests() diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index a3e4545..65e779d 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -14,8 +14,10 @@ import ctypes as _ctypes import subprocess as _subprocess +from dataclasses import dataclass +import string -from typing import cast, Optional +from typing import Optional, Dict, NamedTuple, List, Literal, Set, Any import lib_sonnetconfig @@ -139,6 +141,7 @@ def GenerateCacheFile(fin: str, fout: str) -> None: def ParseDuration(s: str) -> int: """ + Deprecated: use MustParseDuration or ParseDurationSuper Parses a Duration from a string using go stdlib :raises: errors.NoBinaryError - ctypes could not load the go binary, no parsing occurred @@ -160,30 +163,177 @@ def ParseDuration(s: str) -> int: if r.err != 0: raise errors.ParseFailureError(f"ParseDuration: returned status code {r.err}") - return cast(int, r.ret // 1000 // 1000 // 1000) + return int(r.ret // 1000 // 1000 // 1000) def MustParseDuration(s: str) -> int: """ - Parses a Duration from a string using go stdlib, and fallsback to inferior python version if it does not exist - - Most situations should call this over ParseDuration, unless you want to - assert whether the golib exists or not, or avoid parsing in python entirely + Parses a Duration from a string using ParseDurationSuper with API similarity to ParseDuration :raises: errors.ParseFailureError - Failed to parse time :returns: int - Time parsed in seconds """ - if _gotools is not None: - - return ParseDuration(s) + ret = ParseDurationSuper(s) + if ret is None: + raise errors.ParseFailureError("MustParseDuration: pyparser returned error") else: + return ret + + +_super_table: Dict[str, int] = { + "week": 60 * 60 * 24 * 7, + "w": 60 * 60 * 24 * 7, + "day": 60 * 60 * 24, + "d": 60 * 60 * 24, + "hour": 60 * 60, + "h": 60 * 60, + "minute": 60, + "m": 60, + "second": 1, + "s": 1, + } + +_suffix_table: Dict[str, int] = dict([("weeks", 60 * 60 * 24 * 7), ("days", 60 * 60 * 24), ("hours", 60 * 60), ("minutes", 60), ("seconds", 1)] + list(_super_table.items())) + +_alpha_chars = set(char for word in _suffix_table for char in word) +_digit_chars = set(string.digits + ".") +_allowed_chars: Set[str] = _alpha_chars.union(_digit_chars) + + +class _SuffixedNumber(NamedTuple): + number: float + suffix: str + + +@dataclass +class _TypedStr: + __slots__ = "s", "t" + s: str + t: Literal["digit", "suffix"] + + +class _idx_ptr: + __slots__ = "idx", + + def __init__(self, idx: int): + self.idx = idx + + def inc(self) -> "_idx_ptr": + self.idx += 1 + return self + + def __enter__(self) -> "_idx_ptr": + return self + + def __exit__(self, *ignore: Any) -> None: + self.idx += 1 + + +def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: + + tree: List[_TypedStr] = [] + + idx = _idx_ptr(0) + + while idx.idx < len(s): + with idx: + if s[idx.idx] in _digit_chars: + cache = [s[idx.idx]] + while len(s) > idx.idx + 1 and s[idx.idx + 1] in _digit_chars: + cache.append(s[idx.inc().idx]) + + tree.append(_TypedStr("".join(cache), "digit")) + elif s[idx.idx] in _alpha_chars: + + cache = [s[idx.idx]] + while len(s) > idx.idx + 1 and s[idx.idx + 1] in _alpha_chars: + cache.append(s[idx.inc().idx]) + + tree.append(_TypedStr("".join(cache), "suffix")) + + else: + return None + + if not tree: + return None + + if len(tree) == 1: + if tree[0].t == "digit": + try: + if "." in tree[0].s: + return [_SuffixedNumber(float(tree[0].s), "s")] + else: + return [_SuffixedNumber(int(tree[0].s), "s")] + except ValueError: + return None + else: + return None + + # assert len(tree) > 1 + + # Cant start on a suffix + if tree[0].t == "suffix": + return None + + if tree[-1].t == "digit": + return None + + # Assert that lengths are correct, should be asserted by previous logic + if not len(tree) % 2 == 0: + return None + out: List[_SuffixedNumber] = [] + + # alternating digit and suffix starting with digit and ending on suffix + for i in range(0, len(tree), 2): try: - if s[-1] in (multi := {"s": 1, "m": 60, "h": 3600}): - return int(s[:-1]) * multi[s[-1]] + if '.' in tree[i].s: + digit = float(tree[i].s) else: - return int(s) - except (ValueError, TypeError): - raise errors.ParseFailureError("MustParseDuration: pyparser returned error") + digit = int(tree[i].s) + except ValueError: + return None + + if tree[i + 1].s not in _suffix_table: + return None + + out.append(_SuffixedNumber(digit, tree[i + 1].s)) + + return out + + +def ParseDurationSuper(s: str) -> Optional[int]: + """ + Parses a duration in pure python + Allows ({float}{suffix})+ where suffix is weeks|days|hours|minutes|seconds or singular or single char shorthands + Parses {float} => seconds + Parses {singular|single char} suffix => 1 of suffix type (day => 1day) + + :returns: Optional[int] - Return value, None if it could not be parsed + """ + + # Check if in supertable + try: + return _super_table[s] + except KeyError: + pass + + # Quick reject anything not in allowed set + if not all(ch in _allowed_chars for ch in s): + return None + + tree = _str_to_tree(s) + + if tree is None: return None + + out = 0 + + for i in tree: + try: + out += int(i.number * _suffix_table[i.suffix]) + except ValueError: + return None + + return out From 3b95fda3e4843c33cec926d1a97f80af354dfaec Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 15:55:29 -0700 Subject: [PATCH 43/78] add ParseDurationSuper to dunder all --- libs/lib_goparsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 65e779d..5358d18 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -6,6 +6,7 @@ "hascompiled", "ParseDuration", "MustParseDuration", + "ParseDurationSuper", "GenerateCacheFile", "GetVersion", ] From 25384d2091d221651909d15845aa8f90b082c2da Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 16:11:30 -0700 Subject: [PATCH 44/78] Patch fetch_all_mutes error --- libs/lib_sonnetdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/lib_sonnetdb.py b/libs/lib_sonnetdb.py index 2f33aa6..66ecd24 100644 --- a/libs/lib_sonnetdb.py +++ b/libs/lib_sonnetdb.py @@ -516,7 +516,7 @@ def fetch_all_mutes(self) -> List[Tuple[str, str, str, int]]: mute_table: List[Tuple[str, str, str, int]] = [] for i in guild_list: guild_id = str(i[0][:-6]) - for row in self._db.fetch_table(guild_id): + for row in self._db.fetch_table(i[0]): # assert types at runtime infraction_id, user_id, unmute_time = str(row[0]), str(row[1]), int(row[2]) mute_table.append((guild_id, infraction_id, user_id, unmute_time)) From 68049dc98c7d162d36faf3c81497ce462c4b7357 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 16:12:07 -0700 Subject: [PATCH 45/78] add mute length not specified but in reason check --- cmds/cmd_moderation.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index d71d4e7..ee82c52 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -31,7 +31,7 @@ importlib.reload(lib_sonnetcommands) -from lib_goparsers import MustParseDuration +from lib_goparsers import MustParseDuration, ParseDurationSuper from lib_loaders import generate_infractionid, load_embed_color, embed_colors, datetime_now from lib_db_obfuscator import db_hlapi from lib_parsers import parse_user_member, format_duration @@ -466,11 +466,21 @@ async def mute_user(message: discord.Message, args: List[str], client: discord.C # Grab mute time if len(args) >= 2: try: - mutetime = MustParseDuration(args[1]) + mutetime = ParseDurationSuper(args[1]) del args[1] except lib_goparsers.errors.ParseFailureError: - mutetime = 0 + mutetime = None else: + mutetime = None + + misplaced_duration: Optional[str] = None + if mutetime is None: + for i in args[1:]: + if ParseDurationSuper(i) is not None: + misplaced_duration = i + break + + if mutetime is None: mutetime = 0 # This ones for you, curl @@ -502,9 +512,10 @@ async def mute_user(message: discord.Message, args: List[str], client: discord.C return 1 mod_str = f" with {','.join(m.title for m in modifiers)}" if modifiers else "" + duration_str = f"\n(No mute length was specified, but one of the reason items `{misplaced_duration}` is a valid duration, did you mean to mute for this length?)" if misplaced_duration is not None else "" if verbose and not mutetime: - await message.channel.send(f"Muted {member.mention} with ID {member.id}{mod_str} for {reason}", allowed_mentions=discord.AllowedMentions.none()) + await message.channel.send(f"Muted {member.mention} with ID {member.id}{mod_str} for {reason}{duration_str}", allowed_mentions=discord.AllowedMentions.none()) # if mutetime call db timed mute if mutetime: From 1c794de6473c0afe823eb0d590e508680455c6c5 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 16:14:19 -0700 Subject: [PATCH 46/78] patch mute command using new command but being written for old command, causing first reason item to be cut off --- cmds/cmd_moderation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index ee82c52..ed2f600 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -465,11 +465,9 @@ async def mute_user(message: discord.Message, args: List[str], client: discord.C # Grab mute time if len(args) >= 2: - try: - mutetime = ParseDurationSuper(args[1]) + mutetime = ParseDurationSuper(args[1]) + if mutetime is not None: del args[1] - except lib_goparsers.errors.ParseFailureError: - mutetime = None else: mutetime = None From 6927e0a5fa36bcefb85d30003ba11b3e5c0b5860 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 23 Apr 2022 16:15:39 -0700 Subject: [PATCH 47/78] remove unused import --- cmds/cmd_moderation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index ed2f600..fc9cd31 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -31,7 +31,7 @@ importlib.reload(lib_sonnetcommands) -from lib_goparsers import MustParseDuration, ParseDurationSuper +from lib_goparsers import ParseDurationSuper from lib_loaders import generate_infractionid, load_embed_color, embed_colors, datetime_now from lib_db_obfuscator import db_hlapi from lib_parsers import parse_user_member, format_duration From 20fbbb901c1ff7560666e32e35400c137988c02a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 25 Apr 2022 08:54:12 -0700 Subject: [PATCH 48/78] add more words to wordlist --- common/wordlist.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/common/wordlist.txt b/common/wordlist.txt index 2a05cb1..c42e8ed 100644 --- a/common/wordlist.txt +++ b/common/wordlist.txt @@ -104,6 +104,7 @@ air alive alternative always +amaze amazing among ampersand @@ -133,6 +134,7 @@ azure baby back backslash +backspace bake baker bakery @@ -188,6 +190,8 @@ car carbohydrates card cardboard +care +cares cartridge case casting @@ -195,6 +199,7 @@ cat changer channel char +character charisma chat chicken @@ -206,6 +211,7 @@ clean cleaner clear click +closure clothing cloud clouds @@ -213,6 +219,7 @@ clown coal cobblestone coconut +coffee cola collage college @@ -307,9 +314,12 @@ else emerald emulator engineer +english enter equals equation +error +errors eurocentric euroclydon eurodollar @@ -462,6 +472,7 @@ float floppy flowers folding +font food football formula @@ -491,6 +502,7 @@ gift glass glorious glow +go gold good grab @@ -529,10 +541,12 @@ hundred hundredth hyper if +implement imposter in incredible information +infraction input insert inside @@ -554,6 +568,7 @@ invasion invisible ion iron +java jellyfish jupiter justify @@ -563,12 +578,15 @@ keys knishes lab lamb +lambda +language laptop late laugh leather lemon length +letter level library license @@ -781,6 +799,7 @@ octuple odour offer official +ok one onion only @@ -817,6 +836,7 @@ pill pink pizza plain +plane planning plastic play @@ -841,6 +861,7 @@ purple push pyramid pythagoras +python quadrillion quadruple quest @@ -873,6 +894,8 @@ rolling roman rubbish rugby +rune +rust safety sand santa @@ -884,6 +907,7 @@ scissors scorpion screen screwdriver +script seal second sensation @@ -923,8 +947,10 @@ slow slower slowest smelly +snake soccer soda +some someone something sometimes @@ -948,7 +974,9 @@ stone stop stream street +streetcar string +strings structure subscribe subtraction @@ -990,6 +1018,7 @@ toast tomatoes tower train +trait tram transfer trap @@ -997,6 +1026,7 @@ trash treble triangle trio +truck trust tune tuner @@ -1010,6 +1040,7 @@ twist two ultra underscore +unicode union unit unordered From 87e5eb8bd17c45cc993288f6d144969fe07cc15d Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 26 Apr 2022 17:37:22 -0700 Subject: [PATCH 49/78] add fraction parsing to ParseDurationSuper with associated tests --- build_tools/manualtest.py | 7 +++ libs/lib_goparsers.py | 117 +++++++++++++++++++++++++++++--------- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/build_tools/manualtest.py b/build_tools/manualtest.py index c8cdef1..a7bc9ca 100644 --- a/build_tools/manualtest.py +++ b/build_tools/manualtest.py @@ -64,6 +64,13 @@ def test_parse_duration() -> Optional[Iterable[Exception]]: ("3.5d7.3m", 3 * DAY + 12 * HOUR + 7 * MINUTE + (3 * MINUTE // 10)), # Test for fp parse rejection ("5.6.34seconds", None), + # Test fractions + ("3/6days", 12 * HOUR), + ("1/0", None), + ("0/0", None), + ("17/60m", 17 * SECOND), + ("13/24d1/0w", None), + (f"{(1<<54)+2}/2d", (((1 << 54) + 2) * DAY) // 2), ] ) out = [] diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 5358d18..5e709b7 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -18,7 +18,7 @@ from dataclasses import dataclass import string -from typing import Optional, Dict, NamedTuple, List, Literal, Set, Any +from typing import Optional, Dict, List, Literal, Set, Any, Union import lib_sonnetconfig @@ -199,20 +199,48 @@ def MustParseDuration(s: str) -> int: _suffix_table: Dict[str, int] = dict([("weeks", 60 * 60 * 24 * 7), ("days", 60 * 60 * 24), ("hours", 60 * 60), ("minutes", 60), ("seconds", 1)] + list(_super_table.items())) _alpha_chars = set(char for word in _suffix_table for char in word) -_digit_chars = set(string.digits + ".") +_digit_chars = set(string.digits + "./") _allowed_chars: Set[str] = _alpha_chars.union(_digit_chars) -class _SuffixedNumber(NamedTuple): - number: float +@dataclass +class _Fraction: + __slots__ = "numerator", "denominator" + numerator: float + denominator: float + + +@dataclass +class _SuffixedNumber: + __slots__ = "number", "suffix" + number: Union[float, _Fraction] suffix: str + def value(self, multiplier_table: Dict[str, int]) -> Optional[int]: + + multiplier = multiplier_table.get(self.suffix) + + if multiplier is None: + return None + + if isinstance(self.number, (float, int)): + try: + return int(multiplier * self.number) + except ValueError: + return None + + else: + try: + return int((multiplier * self.number.numerator) // self.number.denominator) + except (ValueError, ZeroDivisionError): + return None + @dataclass class _TypedStr: - __slots__ = "s", "t" - s: str - t: Literal["digit", "suffix"] + __slots__ = "val", "typ" + val: str + typ: Literal["digit", "suffix"] class _idx_ptr: @@ -232,6 +260,39 @@ def __exit__(self, *ignore: Any) -> None: self.idx += 1 +def _float_or_int(s: str) -> float: + """ + Raises ValueError on failure + """ + if "." in s: + f = float(s) + if f.is_integer(): + return int(f) + else: + return f + else: + return int(s) + + +def _num_from_typedstr(v: _TypedStr) -> Optional[Union[float, _Fraction]]: + if v.typ == "digit": + try: + if "/" in v.val: + if v.val.count("/") != 1: + return None + + num, denom = v.val.split("/") + + return _Fraction(_float_or_int(num), _float_or_int(denom)) + + else: + return _float_or_int(v.val) + except ValueError: + return None + else: + return None + + def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: tree: List[_TypedStr] = [] @@ -261,24 +322,18 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: return None if len(tree) == 1: - if tree[0].t == "digit": - try: - if "." in tree[0].s: - return [_SuffixedNumber(float(tree[0].s), "s")] - else: - return [_SuffixedNumber(int(tree[0].s), "s")] - except ValueError: - return None + if tree[0].typ == "digit" and (n := _num_from_typedstr(tree[0])) is not None: + return [_SuffixedNumber(n, "s")] else: return None # assert len(tree) > 1 # Cant start on a suffix - if tree[0].t == "suffix": + if tree[0].typ == "suffix": return None - if tree[-1].t == "digit": + if tree[-1].typ == "digit": return None # Assert that lengths are correct, should be asserted by previous logic @@ -289,18 +344,16 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: # alternating digit and suffix starting with digit and ending on suffix for i in range(0, len(tree), 2): - try: - if '.' in tree[i].s: - digit = float(tree[i].s) - else: - digit = int(tree[i].s) - except ValueError: + + digit = _num_from_typedstr(tree[i]) + + if digit is None: return None - if tree[i + 1].s not in _suffix_table: + if tree[i + 1].val not in _suffix_table: return None - out.append(_SuffixedNumber(digit, tree[i + 1].s)) + out.append(_SuffixedNumber(digit, tree[i + 1].val)) return out @@ -308,10 +361,14 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: def ParseDurationSuper(s: str) -> Optional[int]: """ Parses a duration in pure python - Allows ({float}{suffix})+ where suffix is weeks|days|hours|minutes|seconds or singular or single char shorthands - Parses {float} => seconds + Where number is a float or float/float fraction + Allows ({number}{suffix})+ where suffix is weeks|days|hours|minutes|seconds or singular or single char shorthands + Parses {number} => seconds Parses {singular|single char} suffix => 1 of suffix type (day => 1day) + Numbers are kept as integer values when possible, floating point values are subject to IEEE-754 64 bit limitations. + Fraction numerators are multiplied by their suffix multiplier before being divided by their denominator, with floating point math starting where the first float is present + :returns: Optional[int] - Return value, None if it could not be parsed """ @@ -333,7 +390,11 @@ def ParseDurationSuper(s: str) -> Optional[int]: for i in tree: try: - out += int(i.number * _suffix_table[i.suffix]) + v: Optional[int] + if (v := i.value(_suffix_table)) is not None: + out += v + else: + return None except ValueError: return None From af2ab148df806de4e5d956b4e382e5f26f4bf30a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 26 Apr 2022 17:48:00 -0700 Subject: [PATCH 50/78] pin versions to not include breaking change releases --- requirements.extras.txt | 4 ++-- requirements.txt | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.extras.txt b/requirements.extras.txt index 1927cfa..dc48667 100644 --- a/requirements.extras.txt +++ b/requirements.extras.txt @@ -1,2 +1,2 @@ -mariadb >= 1.0.8 -google-re2 >= 0.2.20211101 +mariadb >=1.0.11, <2.0.0 +google-re2 >=0.2.20211101, <1.0.0 diff --git a/requirements.txt b/requirements.txt index 56b0179..33eb9ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -discord.py >= 1.7.3 -discord.py-stubs >= 1.7.3 -cryptography >= 36.0.1 -lz4 >= 3.1.10 -typing-extensions >= 3.7.4 +discord.py >=1.7.3, <2.0 +discord.py-stubs >=1.7.3, <2.0 +cryptography >=37.0.0, <38.0.0 +lz4 >=3.1.10, <5.0.0 +typing-extensions >=3.7.4, <4.0.0 From e7336da7405c422d88ed62a69cf31c1cce27822f Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 26 Apr 2022 17:49:07 -0700 Subject: [PATCH 51/78] swap from `type: ignore` to library stubs with cryptography endpoints --- libs/lib_encryption_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/lib_encryption_wrapper.py b/libs/lib_encryption_wrapper.py index 94bdaaf..02804a0 100644 --- a/libs/lib_encryption_wrapper.py +++ b/libs/lib_encryption_wrapper.py @@ -54,7 +54,7 @@ def __init__(self, filename: Union[bytes, str, _WriteSeekCloser], key: bytes, iv # Start cipher system self.cipher = Cipher(algorithms.AES(key), modes.CTR(iv)) - self.encryptor_module: crypto_typing.encryptor_decryptor = self.cipher.encryptor() # type: ignore[no-untyped-call] + self.encryptor_module = self.cipher.encryptor() # Initialize HMAC generator self.HMACencrypt = hmac.HMAC(key, hashes.SHA512()) @@ -152,7 +152,7 @@ def __init__(self, filename: Union[bytes, str, _ReadSeekCloser], key: bytes, iv: # Make decryptor instance self.cipher = Cipher(algorithms.AES(key), modes.CTR(iv)) - self.decryptor_module: crypto_typing.encryptor_decryptor = self.cipher.decryptor() # type: ignore[no-untyped-call] + self.decryptor_module = self.cipher.decryptor() # Generate HMAC HMACobj = hmac.HMAC(key, hashes.SHA512()) From 5d23682381b5a5a381422db3f042b504acf516af Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 27 Apr 2022 09:07:17 -0700 Subject: [PATCH 52/78] add sanity limit to infraction generator retrys --- cmds/cmd_moderation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index fc9cd31..01bb9ee 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -102,9 +102,15 @@ async def log_infraction( with db_hlapi(message.guild.id) as db: + iterations: int = 0 + iter_limit: Final[int] = 10_000 # Infraction id collision test while db.grab_infraction(generated_id := generate_infractionid()): - continue + iterations += 1 + if iterations > iter_limit: + raise lib_sonnetcommands.CommandError( + "ERROR: Failed to generate a unique infraction ID after {iter_limit} attempts\n(Do you have too many infractions/too small of a wordlist installed?)" + ) # Grab log channel try: From eb75bdc406c7c59550d537832a2e9e30b6ab7e6e Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 27 Apr 2022 13:10:11 -0700 Subject: [PATCH 53/78] swap build tests to use manualtest --- .github/workflows/python-dev.yml | 4 ++-- .github/workflows/python-package.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a048566..f7b73b6 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -43,9 +43,9 @@ jobs: - name: check syntax integrity run: | python build_tools/testimport.py - - name: check ramfs integrity + - name: check manual tests run: | - python build_tools/ramfsassert.py + python build_tools/manualtest.py - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index eaaab79..23fa558 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -44,9 +44,9 @@ jobs: - name: check syntax integrity run: | python build_tools/testimport.py - - name: check ramfs integrity + - name: check manual tests run: | - python build_tools/ramfsassert.py + python build_tools/manualtest.py - name: run mypy type checking run: | mypy . --ignore-missing-imports --strict --warn-unreachable From 1df0b944aac1c10fd10ab90477e3788164242040 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Wed, 27 Apr 2022 13:10:33 -0700 Subject: [PATCH 54/78] move ramfsassert into manualtest lib --- build_tools/manualtest.py | 63 +++++++++++++++++++++++++++++++++++--- build_tools/ramfsassert.py | 45 --------------------------- 2 files changed, 59 insertions(+), 49 deletions(-) delete mode 100644 build_tools/ramfsassert.py diff --git a/build_tools/manualtest.py b/build_tools/manualtest.py index a7bc9ca..3241fa2 100644 --- a/build_tools/manualtest.py +++ b/build_tools/manualtest.py @@ -1,10 +1,10 @@ import sys import os +import traceback sys.path.insert(1, os.getcwd() + '/common') sys.path.insert(1, os.getcwd() + '/libs') - -from lib_goparsers import ParseDurationSuper +sys.path.insert(1, os.getcwd()) from typing import Callable, TypeVar, List, Optional, Final, Iterable @@ -17,6 +17,9 @@ def test_func_io(func: Callable[[T], O], arg: T, expect: O) -> None: def test_parse_duration() -> Optional[Iterable[Exception]]: + + from lib_goparsers import ParseDurationSuper + WEEK: Final = 7 * 24 * 60 * 60 DAY: Final = 24 * 60 * 60 HOUR: Final = 60 * 60 @@ -86,7 +89,59 @@ def test_parse_duration() -> Optional[Iterable[Exception]]: return None -testfuncs: List[Callable[[], Optional[Iterable[Exception]]]] = [test_parse_duration] +def test_ramfs() -> Optional[Iterable[Exception]]: + + from contextlib import redirect_stdout, redirect_stderr + + sink = open("/dev/null", "w") + + # Reroute stderr and stdout to ignore import warnings from main + with redirect_stdout(sink): + with redirect_stderr(sink): + from main import ram_filesystem # pylint: disable=E0401 + + testfs = ram_filesystem() + errs = [] + + def assertdir(files: List[str], directory: List[str]) -> None: + try: + assert testfs.ls() == (files, directory), f"{testfs.ls()=} != {(files, directory)=}" + except AssertionError as e: + errs.append(e) + + testfs.mkdir("abcde") + + assertdir([], ["abcde"]) + + testfs.rmdir("abcde") + + assertdir([], []) + + testfs.create_f("dir/file", f_type=bytes, f_args=[64]) + + try: + assert isinstance(testfs.read_f("dir/file"), bytes) and len(testfs.read_f("dir/file")) == 64 + except AssertionError as e: + errs.append(e) + + assertdir([], ["dir"]) + + testfs.remove_f("dir/file") + + try: + assert testfs.ls("dir") == ([], []) + except AssertionError as e: + errs.append(e) + + testfs.rmdir("dir") + + assertdir([], []) + + if errs: return errs + else: return None + + +testfuncs: List[Callable[[], Optional[Iterable[Exception]]]] = [test_parse_duration, test_ramfs] def main_tests() -> None: @@ -99,7 +154,7 @@ def main_tests() -> None: failure = True print(i) for e in errs: - print(e) + traceback.print_exception(type(e), e, e.__traceback__) if failure: sys.exit(1) diff --git a/build_tools/ramfsassert.py b/build_tools/ramfsassert.py deleted file mode 100644 index 471d39e..0000000 --- a/build_tools/ramfsassert.py +++ /dev/null @@ -1,45 +0,0 @@ -# Tool to test ramfs correctness -# Ultrabear 2021 - -# imports LeXdPyK, hopefully -# This is jank - -import sys -import os - -sys.path.insert(1, os.getcwd()) - -from main import ram_filesystem # pylint: disable=E0401 - -from typing import List - -testfs = ram_filesystem() - - -def assertdir(files: List[str], directory: List[str]) -> None: - assert testfs.ls() == (files, directory) - - -testfs.mkdir("abcde") - -assertdir([], ["abcde"]) - -testfs.rmdir("abcde") - -assertdir([], []) - -testfs.create_f("dir/file", f_type=bytes, f_args=[64]) - -assert isinstance(testfs.read_f("dir/file"), bytes) and len(testfs.read_f("dir/file")) == 64 - -assertdir([], ["dir"]) - -testfs.remove_f("dir/file") - -assert testfs.ls("dir") == ([], []) - -testfs.rmdir("dir") - -assertdir([], []) - -print("Testing Completed with no errors") From 006ae26b24f69c01c11540ac43c6acd419ce0c29 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Fri, 29 Apr 2022 18:26:24 -0700 Subject: [PATCH 55/78] memoize ParseDurationSuper to aid in mute processing --- libs/lib_goparsers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 5e709b7..65d2da6 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -17,6 +17,7 @@ import subprocess as _subprocess from dataclasses import dataclass import string +from functools import lru_cache from typing import Optional, Dict, List, Literal, Set, Any, Union @@ -358,6 +359,7 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: return out +@lru_cache(maxsize=500) def ParseDurationSuper(s: str) -> Optional[int]: """ Parses a duration in pure python From eda2df9dd027ec2f4b91894c26264d93821ce541 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 3 May 2022 10:14:52 -0700 Subject: [PATCH 56/78] add try_or_return wrapper around raises --- build_tools/manualtest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/build_tools/manualtest.py b/build_tools/manualtest.py index 3241fa2..3beaa2c 100644 --- a/build_tools/manualtest.py +++ b/build_tools/manualtest.py @@ -16,6 +16,17 @@ def test_func_io(func: Callable[[T], O], arg: T, expect: O) -> None: assert func(arg) == expect, f"func({arg=})={func(arg)} != {expect=}" +def try_or_return(func: Callable[[], Optional[Iterable[Exception]]]) -> Callable[[], Optional[Iterable[Exception]]]: + def wrapped() -> Optional[Iterable[Exception]]: + try: + return func() + except Exception as e: + return [e] + + return wrapped + + +@try_or_return def test_parse_duration() -> Optional[Iterable[Exception]]: from lib_goparsers import ParseDurationSuper @@ -89,6 +100,7 @@ def test_parse_duration() -> Optional[Iterable[Exception]]: return None +@try_or_return def test_ramfs() -> Optional[Iterable[Exception]]: from contextlib import redirect_stdout, redirect_stderr From fc5636d74a84702aee3714ea1e6f70b50dec8fdb Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 3 May 2022 13:36:23 -0700 Subject: [PATCH 57/78] make autotest say how long each item takes and print each item as soon as it completes --- build_tools/autotest.py | 45 ++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/build_tools/autotest.py b/build_tools/autotest.py index ad85451..db47a80 100644 --- a/build_tools/autotest.py +++ b/build_tools/autotest.py @@ -1,10 +1,12 @@ import threading import sys import subprocess +import time +from queue import Queue from dataclasses import dataclass -from typing import Dict, Optional, Union, List +from typing import Dict, Union, List, Iterator # Wrapper around string to make the command run in sh instead of pexec @@ -18,6 +20,7 @@ class Color: red = "\033[91m" blue = "\033[94m" reset = "\033[0m" + green = "\033[92m" @dataclass @@ -25,42 +28,52 @@ class RunningProc: args: str stdout: bytes stderr: bytes - thread: Optional[threading.Thread] + duration_ns: int -def run(args: Union[List[str], str], shell: bool, hm: RunningProc) -> None: +def into_str(args: Union[List[str], str]) -> str: + + if isinstance(args, list): + return " ".join(args) + + return args + + +def run(args: Union[List[str], str], shell: bool, q: "Queue[RunningProc]") -> None: + start = time.monotonic_ns() ret = subprocess.run(args, shell=shell, capture_output=True) - hm.stdout = ret.stdout - hm.stderr = ret.stderr + q.put(RunningProc(into_str(args), ret.stdout, ret.stderr, time.monotonic_ns() - start)) -def initjobs(tests: Dict[str, Union[str, Shell]]) -> Dict[str, RunningProc]: - testout: Dict[str, RunningProc] = {} +def initjobs(tests: Dict[str, Union[str, Shell]]) -> "Queue[RunningProc]": + + testout: "Queue[RunningProc]" = Queue(maxsize=len(tests)) for k, v in tests.items(): - testout[k] = RunningProc(v, b"", b"", None) flag = isinstance(v, Shell) args = v if flag else v.split() - t = threading.Thread(target=run, args=(args, flag, testout[k])) - testout[k].thread = t + t = threading.Thread(target=run, args=(args, flag, testout)) t.start() return testout -def finishjobs(testout: Dict[str, RunningProc]) -> None: +def lim_yield(q: "Queue[RunningProc]", lim: int) -> Iterator[RunningProc]: + + for _ in range(lim): + yield q.get() + - for _, v in testout.items(): - assert v.thread is not None +def finishjobs(testout: "Queue[RunningProc]", tests_c: int) -> None: - v.thread.join() + for v in lim_yield(testout, tests_c): err = v.stdout.decode("utf8") + v.stderr.decode("utf8") - cmdfmt = f"{Color.blue}{v.args}{Color.reset}" + cmdfmt = f"{Color.blue}{v.args}{Color.green} {v.duration_ns//1000//1000}ms{Color.reset}" isshell = f'{Color.red}sh -c ' * isinstance(v.args, Shell) print(isshell + cmdfmt) @@ -84,7 +97,7 @@ def main() -> None: if i in nottest: del tests[i] - finishjobs(initjobs(tests)) + finishjobs(initjobs(tests), len(tests)) if __name__ == "__main__": From c8cfc7e9a04cbda84e7f8f42b822f5a00fb898b8 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 3 May 2022 17:04:12 -0700 Subject: [PATCH 58/78] remove unix specific call in manualtest.py --- build_tools/manualtest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_tools/manualtest.py b/build_tools/manualtest.py index 3beaa2c..28d2332 100644 --- a/build_tools/manualtest.py +++ b/build_tools/manualtest.py @@ -1,6 +1,7 @@ import sys import os import traceback +import io sys.path.insert(1, os.getcwd() + '/common') sys.path.insert(1, os.getcwd() + '/libs') @@ -105,7 +106,7 @@ def test_ramfs() -> Optional[Iterable[Exception]]: from contextlib import redirect_stdout, redirect_stderr - sink = open("/dev/null", "w") + sink = io.StringIO() # Reroute stderr and stdout to ignore import warnings from main with redirect_stdout(sink): From a7c0515ce74b233c87ad1b2e47fc5d9f3bdc1a39 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 3 May 2022 17:36:58 -0700 Subject: [PATCH 59/78] fix spelling errors found by codespell --- cmds/cmd_automod.py | 2 +- dlibs/dlib_userupdate.py | 4 ++-- libs/lib_goparsers.py | 2 +- libs/lib_tparse.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmds/cmd_automod.py b/cmds/cmd_automod.py index 1ac34a3..29dee62 100644 --- a/cmds/cmd_automod.py +++ b/cmds/cmd_automod.py @@ -242,7 +242,7 @@ async def list_blacklist(message: discord.Message, args: List[str], client: disc if blacklist[i]: blacklist[i] = [blacklist[i]] - # If not existant then get rid of it + # If not existent then get rid of it for i in list(blacklist.keys()): if not blacklist[i]: del blacklist[i] diff --git a/dlibs/dlib_userupdate.py b/dlibs/dlib_userupdate.py index 0f7ad3e..b9c793a 100644 --- a/dlibs/dlib_userupdate.py +++ b/dlibs/dlib_userupdate.py @@ -129,7 +129,7 @@ async def on_member_join(member: discord.Member, **kargs: Any) -> None: issues: List[str] = [] - # Handle notifer logging + # Handle notifier logging if member.id in notifier_cache["notifier-log-users"]: issues.append("User") if abs(discord_datetime_now().timestamp() - member.created_at.timestamp()) < int(notifier_cache["notifier-log-timestamp"]): @@ -196,4 +196,4 @@ async def on_member_remove(member: discord.Member, **kargs: Any) -> None: "on-member-remove": on_member_remove, } -version_info: str = "1.2.10" +version_info: str = "1.2.13-DEV" diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 65d2da6..b4cd8c3 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -330,7 +330,7 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: # assert len(tree) > 1 - # Cant start on a suffix + # Can't start on a suffix if tree[0].typ == "suffix": return None diff --git a/libs/lib_tparse.py b/libs/lib_tparse.py index 71f4d16..d44f8e4 100644 --- a/libs/lib_tparse.py +++ b/libs/lib_tparse.py @@ -213,7 +213,7 @@ def parse(self, args: List[str] = sys.argv[1:], stderr: StringWriter = sys.stder self._error(f"Failed to parse argument {val}; ValueError: {ve}", exit_on_fail, stderr) elif not lazy: - self._error("Recieved garbage argument", exit_on_fail, stderr) + self._error("Received garbage argument", exit_on_fail, stderr) if consume: garbage.reverse() From 67868e33a78b8122e7605afea96a8213bd1b6706 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 3 May 2022 17:40:52 -0700 Subject: [PATCH 60/78] fix errors found by codespell --- BANNED.md | 2 +- cmds/cmd_utils.py | 4 ++-- cmds/cmd_version.py | 4 ++-- common/wordlist.txt | 4 ++-- libs/lib_parsers.py | 6 +++--- libs/lib_sonnetcommands.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/BANNED.md b/BANNED.md index 884c6a3..4dc3ffc 100644 --- a/BANNED.md +++ b/BANNED.md @@ -4,7 +4,7 @@ None of the things listed below should happen in Sonnet. # Absolutely not - Do not have blind excepts, this means a `try:` `except:` statement with no specific error to catch - Do not have try excepts that cover more than one or a few lines - - This can be ommited if the errors caught are custom errors or errors that will only be on absolutely known lines + - This can be omitted if the errors caught are custom errors or errors that will only be on absolutely known lines - Do not use `input()` or `print()` unless it is for debug or exceptions - Do not use `input()` even for debugging, it blocks asyncio - Respect asyncio, do not use threading or multiprocessing, they are not designed to work together and introduce bugs diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 15b1af2..0774524 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -276,7 +276,7 @@ async def full_help(self, page: int, per_page: int) -> discord.Embed: cmd_embed.set_author(name=self.helpname) total = 0 - # Total counting is seperate due to pagination not counting all modules + # Total counting is separate due to pagination not counting all modules for cmd in cmds_dict: if 'alias' not in cmds_dict[cmd]: total += 1 @@ -356,7 +356,7 @@ async def help_function(message: discord.Message, args: List[str], client: disco # Do not echo user input else: - # lets check if they cant read documentation + # lets check if they can't read documentation probably_tried_paging: bool try: probably_tried_paging = int(args[0]) <= ((len(cmds) + (per_page - 1)) // per_page) diff --git a/cmds/cmd_version.py b/cmds/cmd_version.py index 7770265..57254b7 100644 --- a/cmds/cmd_version.py +++ b/cmds/cmd_version.py @@ -137,7 +137,7 @@ async def print_stats(message: discord.Message, args: List[str], client: discord for i in global_statistics_file: outputmap.append([i, global_statistics_file[i]]) - # Declare here cause fstrings cant have \ in it 草 + # Declare here cause fstrings can't have \ in it 草 newline = "\n" writer = io.StringIO() @@ -184,4 +184,4 @@ async def print_stats(message: discord.Message, args: List[str], client: discord } } -version_info: str = "1.2.12" +version_info: str = "1.2.13-DEV" diff --git a/common/wordlist.txt b/common/wordlist.txt index c42e8ed..052d1c6 100644 --- a/common/wordlist.txt +++ b/common/wordlist.txt @@ -654,11 +654,11 @@ next night nine nineteen -nineth +ninth ninetieth ninety ninja -ninteenth +nineteenth no nondurability nondurable diff --git a/libs/lib_parsers.py b/libs/lib_parsers.py index ed5858e..dec37e3 100644 --- a/libs/lib_parsers.py +++ b/libs/lib_parsers.py @@ -74,7 +74,7 @@ def _formatregexfind(gex: List[Any]) -> str: return ", ".join(i if isinstance(i, str) else "".join(i) for i in gex) -# This exists because type checkers cant infer lambda return types or something +# This exists because type checkers can't infer lambda return types or something def returnsNone() -> None: ... @@ -85,7 +85,7 @@ def parse_blacklist(indata: _parse_blacklist_inputs) -> tuple[bool, bool, list[s Deprecated, this should be in dlib_messages.py Parse the blacklist over a message object - :returns: Tuple[bool, bool, List[str]] -- broke blacklist, broke notifer list, list of strings of infraction messages + :returns: Tuple[bool, bool, List[str]] -- broke blacklist, broke notifier list, list of strings of infraction messages """ message, blacklist, ramfs = indata @@ -809,7 +809,7 @@ def paginate_noexcept(vals: List[_PT], page: int, per_page: int, lim: int, fmtfu pospool = sum(i for i in lenarr if i > 0) # Remove negatives newmaxlen = maxlen + (pospool // actual_per_page) # Account for per item in our new pospool if newmaxlen <= 1: - raise lib_sonnetcommands.CommandError("ERROR: The amount of items to display overflows the set possible limit with newline seperators") + raise lib_sonnetcommands.CommandError("ERROR: The amount of items to display overflows the set possible limit with newline separators") for i in itemstore: # Cap at newmaxlen-1 and then add \n at the end diff --git a/libs/lib_sonnetcommands.py b/libs/lib_sonnetcommands.py index 87d7b28..c5c1002 100644 --- a/libs/lib_sonnetcommands.py +++ b/libs/lib_sonnetcommands.py @@ -162,7 +162,7 @@ class SonnetCommand(dict): # type: ignore[type-arg] def __init__(self, vals: Dict[str, Any], aliasmap: Optional[Dict[str, Dict[str, Any]]] = None) -> None: """ Init a new sonnetcommand instance. - If an aliasmap is passed, the command will be checked for if it has an alias and inheret it into itself if it does + If an aliasmap is passed, the command will be checked for if it has an alias and inherit it into itself if it does """ if aliasmap is not None and 'alias' in vals: vals = aliasmap[vals['alias']] From a94e3ed170dac68d1e77edea4a67bd38c49bb396 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 5 May 2022 08:38:27 -0700 Subject: [PATCH 61/78] LeXdPyK 1.4.12: change var name to satisfy pyright errors --- main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 703a6a3..22d16cc 100644 --- a/main.py +++ b/main.py @@ -581,6 +581,7 @@ async def safety_check(guild: Optional[discord.Guild] = None, guild_id: Optional if guild: guild_id = guild.id if user: user_id = user.id + non_null_guild: discord.Guild if user_id and user_id in blacklist["user"] and guild_id: @@ -590,17 +591,17 @@ async def safety_check(guild: Optional[discord.Guild] = None, guild_id: Optional return False try: - guild = await Client.fetch_guild(guild_id) + non_null_guild = await Client.fetch_guild(guild_id) except discord.errors.HTTPException: return False try: - await guild.ban(user, reason="LeXdPyK: SYSTEM LEVEL BLACKLIST", delete_message_days=0) + await non_null_guild.ban(user, reason="LeXdPyK: SYSTEM LEVEL BLACKLIST", delete_message_days=0) except discord.errors.Forbidden: blacklist["guild"].append(guild_id) try: - await guild.leave() + await non_null_guild.leave() return False except discord.errors.HTTPException: pass @@ -613,12 +614,12 @@ async def safety_check(guild: Optional[discord.Guild] = None, guild_id: Optional if guild_id and guild_id in blacklist["guild"]: try: - guild = await Client.fetch_guild(guild_id) + non_null_guild = await Client.fetch_guild(guild_id) except discord.errors.HTTPException: return False try: - await guild.leave() + await non_null_guild.leave() return False except discord.errors.HTTPException: pass @@ -901,7 +902,7 @@ def main(args: List[str]) -> int: # Define version info and start time -version_info: str = "LeXdPyK 1.4.11" +version_info: str = "LeXdPyK 1.4.12" bot_start_time: float = time.time() if __name__ == "__main__": From e72bdedc6f1fb7d36ef938b55949bb74bf0241ef Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 5 May 2022 09:11:13 -0700 Subject: [PATCH 62/78] make autotest return 1 on a command erroring --- build_tools/autotest.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/build_tools/autotest.py b/build_tools/autotest.py index db47a80..c58868c 100644 --- a/build_tools/autotest.py +++ b/build_tools/autotest.py @@ -29,6 +29,7 @@ class RunningProc: stdout: bytes stderr: bytes duration_ns: int + returncode: int def into_str(args: Union[List[str], str]) -> str: @@ -43,7 +44,7 @@ def run(args: Union[List[str], str], shell: bool, q: "Queue[RunningProc]") -> No start = time.monotonic_ns() ret = subprocess.run(args, shell=shell, capture_output=True) - q.put(RunningProc(into_str(args), ret.stdout, ret.stderr, time.monotonic_ns() - start)) + q.put(RunningProc(into_str(args), ret.stdout, ret.stderr, time.monotonic_ns() - start, ret.returncode)) def initjobs(tests: Dict[str, Union[str, Shell]]) -> "Queue[RunningProc]": @@ -67,7 +68,9 @@ def lim_yield(q: "Queue[RunningProc]", lim: int) -> Iterator[RunningProc]: yield q.get() -def finishjobs(testout: "Queue[RunningProc]", tests_c: int) -> None: +def finishjobs(testout: "Queue[RunningProc]", tests_c: int) -> int: + + errno = 0 for v in lim_yield(testout, tests_c): @@ -79,8 +82,13 @@ def finishjobs(testout: "Queue[RunningProc]", tests_c: int) -> None: print(isshell + cmdfmt) if err: print(err, end="") + if v.returncode != 0: + errno = 1 + + return errno + -def main() -> None: +def main() -> int: tests: Dict[str, Union[str, Shell]] = { "pyflakes": "pyflakes .", @@ -97,8 +105,8 @@ def main() -> None: if i in nottest: del tests[i] - finishjobs(initjobs(tests), len(tests)) + return finishjobs(initjobs(tests), len(tests)) if __name__ == "__main__": - main() + sys.exit(main()) From f788b0c8a4a5ccb0b90ca4f50635035c736bbdb7 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 7 May 2022 10:52:20 -0700 Subject: [PATCH 63/78] fix typing_extension requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 33eb9ca..308a2a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ discord.py >=1.7.3, <2.0 discord.py-stubs >=1.7.3, <2.0 cryptography >=37.0.0, <38.0.0 lz4 >=3.1.10, <5.0.0 -typing-extensions >=3.7.4, <4.0.0 +typing-extensions >=3.10.0.0, <5 From 8332f65751e417913ccbafe1492cf0c7e5cf9e0b Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 9 May 2022 08:24:50 -0700 Subject: [PATCH 64/78] make pyright use correct ignore for single line --- libs/lib_sonnetconfig.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/lib_sonnetconfig.py b/libs/lib_sonnetconfig.py index b9279fe..911a995 100644 --- a/libs/lib_sonnetconfig.py +++ b/libs/lib_sonnetconfig.py @@ -41,9 +41,7 @@ def _load_cfg(attr: str, default: Typ, typ: Type[Typ], testfunc: Optional[Callab raise TypeError(f"Sonnet Config {attr}: {errmsg}") # pyright thinks that it can still be Any despite isinstance check - # This applies to the whole file but mypy interferes with type: ignore syntax ;-; - # pyright: reportGeneralTypeIssues=false - return conf + return conf # pyright: ignore[reportGeneralTypeIssues] # Prints a warning if not using re2 From da41c25e68e756369869926e720d3ce37e894479 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Thu, 12 May 2022 17:26:51 -0700 Subject: [PATCH 65/78] make duration parser not accept non prefixed fractions --- libs/lib_goparsers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index b4cd8c3..63aae06 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -324,7 +324,10 @@ def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: if len(tree) == 1: if tree[0].typ == "digit" and (n := _num_from_typedstr(tree[0])) is not None: - return [_SuffixedNumber(n, "s")] + # disallow fractions as non prefixed numbers + if not isinstance(n, _Fraction): + return [_SuffixedNumber(n, "s")] + return None else: return None From e7f298676bd70ab61c18ee1fa636e4db5aef27ca Mon Sep 17 00:00:00 2001 From: ultrabear Date: Sat, 2 Jul 2022 20:10:56 -0700 Subject: [PATCH 66/78] patch typing issue found by pyright --- libs/lib_mdb_handler.py | 2 +- libs/lib_sonnetdb.py | 2 +- libs/lib_sql_handler.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/lib_mdb_handler.py b/libs/lib_mdb_handler.py index 9ce685a..d51dd21 100644 --- a/libs/lib_mdb_handler.py +++ b/libs/lib_mdb_handler.py @@ -172,7 +172,7 @@ def fetch_table(self, table: str) -> Tuple[Any, ...]: returndata = tuple(self.cur) return returndata - def list_tables(self, searchterm: str) -> Tuple[Any, ...]: + def list_tables(self, searchterm: str) -> Tuple[Tuple[str], ...]: self.cur.execute(f"SHOW TABLES WHERE Tables_in_{self.db_name} LIKE ?", (searchterm, )) diff --git a/libs/lib_sonnetdb.py b/libs/lib_sonnetdb.py index 66ecd24..c8ab83b 100644 --- a/libs/lib_sonnetdb.py +++ b/libs/lib_sonnetdb.py @@ -511,7 +511,7 @@ def fetch_all_mutes(self) -> List[Tuple[str, str, str, int]]: """ # Grab list of tables - guild_list: Tuple[List[str], ...] = self._db.list_tables("%_mutes") + guild_list: Tuple[Tuple[str], ...] = self._db.list_tables("%_mutes") mute_table: List[Tuple[str, str, str, int]] = [] for i in guild_list: diff --git a/libs/lib_sql_handler.py b/libs/lib_sql_handler.py index 7a74a98..3ee690e 100644 --- a/libs/lib_sql_handler.py +++ b/libs/lib_sql_handler.py @@ -199,7 +199,7 @@ def fetch_table(self, table: str) -> Tuple[Any, ...]: # Fetches a full table # Send data return tuple(self.cur.fetchall()) - def list_tables(self, searchterm: str) -> Tuple[str, ...]: + def list_tables(self, searchterm: str) -> Tuple[Tuple[str], ...]: self.cur.execute("SELECT name FROM sqlite_master WHERE name LIKE ?;", (searchterm, )) From 2d43fd450cb241d32ba0baad416a71e6887b4117 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 4 Jul 2022 15:31:07 -0700 Subject: [PATCH 67/78] add datetimeplus library --- libs/lib_datetimeplus.py | 583 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 583 insertions(+) create mode 100644 libs/lib_datetimeplus.py diff --git a/libs/lib_datetimeplus.py b/libs/lib_datetimeplus.py new file mode 100644 index 0000000..7b71543 --- /dev/null +++ b/libs/lib_datetimeplus.py @@ -0,0 +1,583 @@ +# A nice time api? +# constructed based off of the idea that datetime is a sound library with a bad api, uses datetime primitives internally but provides a simpler api +# Ultrabear 2022 + +import datetime +import time +import copy as pycopy +from dataclasses import dataclass +import string + +from typing import Optional, Final, List, Any, Union, Literal, Set, Dict + +__all__ = [ + "Time", + "Clock", + "Date", + "Duration", + ] + +_NANOS_PER_SECOND: Final = 1000 * 1000 * 1000 +_NANOS_PER_MILLI: Final = 1000 * 1000 +_NANOS_PER_MICRO: Final = 1000 + +# parse method data +_super_table: Dict[str, int] = { + "week": 60 * 60 * 24 * 7, + "w": 60 * 60 * 24 * 7, + "day": 60 * 60 * 24, + "d": 60 * 60 * 24, + "hour": 60 * 60, + "h": 60 * 60, + "minute": 60, + "m": 60, + "second": 1, + "s": 1, + } + +_suffix_table: Dict[str, int] = dict([("weeks", 60 * 60 * 24 * 7), ("days", 60 * 60 * 24), ("hours", 60 * 60), ("minutes", 60), ("seconds", 1)] + list(_super_table.items())) + +_alpha_chars = set(char for word in _suffix_table for char in word) +_digit_chars = set(string.digits + "./") +_allowed_chars: Set[str] = _alpha_chars.union(_digit_chars) + + +@dataclass +class _Fraction: + """ + A pure fraction with a numerator and denominator + """ + __slots__ = "numerator", "denominator" + numerator: float + denominator: float + + +@dataclass +class _SuffixedNumber: + """ + A suffixed number construct where the number is either a int, float, or fractional representation + the suffix maps to a multiplier table (i/e 1 minute -> 60 seconds) + """ + __slots__ = "number", "suffix" + number: Union[float, _Fraction] + suffix: str + + def value(self, multiplier_table: Dict[str, int]) -> Optional[int]: + + multiplier = multiplier_table.get(self.suffix) + + if multiplier is None: + return None + + if isinstance(self.number, (float, int)): + try: + return int(multiplier * self.number) + except ValueError: + return None + + else: + try: + return int((multiplier * self.number.numerator) // self.number.denominator) + except (ValueError, ZeroDivisionError): + return None + + +@dataclass +class _TypedStr: + """ + A typed string in the preprocessing stage, has not yet been processed as a suffixed number + but has been typed as a number or suffix + """ + __slots__ = "val", "typ" + val: str + typ: Literal["digit", "suffix"] + + +class _idx_ptr: + """ + Helper method to wrap increments of indexes + """ + __slots__ = "idx", + + def __init__(self, idx: int): + self.idx = idx + + def inc(self) -> "_idx_ptr": + self.idx += 1 + return self + + def __enter__(self) -> "_idx_ptr": + return self + + def __exit__(self, *ignore: Any) -> None: + self.idx += 1 + + +def _float_or_int(s: str) -> float: + """ + Raises ValueError on failure + """ + if "." in s: + f = float(s) + if f.is_integer(): + return int(f) + else: + return f + else: + return int(s) + + +def _num_from_typedstr(v: _TypedStr) -> Optional[Union[float, _Fraction]]: + """ + Returns a number from a digit typed string + """ + if v.typ == "digit": + try: + if "/" in v.val: + if v.val.count("/") != 1: + return None + + num, denom = v.val.split("/") + + return _Fraction(_float_or_int(num), _float_or_int(denom)) + + else: + return _float_or_int(v.val) + except ValueError: + return None + else: + return None + + +def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: + """ + Converts a string into a list of suffixed numbers + """ + + tree: List[_TypedStr] = [] + + idx = _idx_ptr(0) + + while idx.idx < len(s): + with idx: + if s[idx.idx] in _digit_chars: + cache = [s[idx.idx]] + while len(s) > idx.idx + 1 and s[idx.idx + 1] in _digit_chars: + cache.append(s[idx.inc().idx]) + + tree.append(_TypedStr("".join(cache), "digit")) + elif s[idx.idx] in _alpha_chars: + + cache = [s[idx.idx]] + while len(s) > idx.idx + 1 and s[idx.idx + 1] in _alpha_chars: + cache.append(s[idx.inc().idx]) + + tree.append(_TypedStr("".join(cache), "suffix")) + + else: + return None + + if not tree: + return None + + if len(tree) == 1: + if tree[0].typ == "digit" and (n := _num_from_typedstr(tree[0])) is not None: + # disallow fractions as non prefixed numbers + if not isinstance(n, _Fraction): + return [_SuffixedNumber(n, "s")] + return None + else: + return None + + # assert len(tree) > 1 + + # Can't start on a suffix + if tree[0].typ == "suffix": + return None + + if tree[-1].typ == "digit": + return None + + # Assert that lengths are correct, should be asserted by previous logic + if not len(tree) % 2 == 0: + return None + + out: List[_SuffixedNumber] = [] + + # alternating digit and suffix starting with digit and ending on suffix + for i in range(0, len(tree), 2): + + digit = _num_from_typedstr(tree[i]) + + if digit is None: + return None + + if tree[i + 1].val not in _suffix_table: + return None + + out.append(_SuffixedNumber(digit, tree[i + 1].val)) + + return out + + +@dataclass +class Clock: + """ + A dataclass representing a wall clock reading of hours minutes seconds and nanoseconds in military time + mostly equivalent to a 'time' or 'clock' from libraries representing time as a date+time + """ + hours: int + minutes: int + seconds: int + nanoseconds: int + + +@dataclass +class Date: + """ + A dataclass representing a date in terms of years, months, and days + mostly equivalent to a 'date' from libraries representing time as a date+time + """ + years: int + months: int + days: int + + +class Duration(int): + def nanos(self) -> int: + """ + Returns duration as nanoseconds + """ + return self + + def micros(self) -> int: + """ + Returns duration as microseconds + """ + return self // _NANOS_PER_MICRO + + def millis(self) -> int: + """ + Returns duration as milliseconds + """ + return self // _NANOS_PER_MILLI + + def seconds(self) -> int: + """ + Returns duration as seconds + """ + return self // _NANOS_PER_SECOND + + def clock(self) -> Clock: + """ + Returns duration as a Clock representation + """ + + nanos = self.nanos() + + seconds = nanos // _NANOS_PER_SECOND + nanos %= _NANOS_PER_SECOND + + minutes = seconds // 60 + seconds %= 60 + + hours = minutes // 60 + minutes %= 60 + + return Clock(hours, minutes, seconds, nanos) + + @classmethod + def from_clock(cls, clock: Clock) -> "Duration": + """ + Constructs a Duration from a Clock object + """ + return cls.from_hms(clock.hours, clock.minutes, clock.seconds, clock.nanoseconds) + + @classmethod + def from_hms(cls, hours: int, minutes: int, seconds: int, nanos: Optional[int] = None) -> "Duration": + """ + Constructs a Duration from hours minutes and seconds, and optionally nanoseconds + Lower types will overflow into larger constructs (i/e 120 seconds will add up to 2 minutes added to the total duration) + """ + + if nanos is None: + nanos = 0 + + minutes += hours * 60 + seconds += minutes * 60 + nanos += seconds * _NANOS_PER_SECOND + + return cls(nanos) + + @classmethod + def from_seconds(cls, seconds: int) -> "Duration": + return cls(seconds * _NANOS_PER_SECOND) + + @classmethod + def parse(cls, s: str) -> Optional["Duration"]: + """ + Parses a Duration up to seconds accuracy + Where number is a float or float/float fraction: + Allows ({number}{suffix})+ where suffix is weeks|days|hours|minutes|seconds or singular or single char shorthands + Parses {number} => seconds + Parses {singular|single char} suffix => 1 of suffix type (day => 1day) + + Numbers are kept as integer values when possible, floating point values are subject to IEEE-754 64 bit limitations. + Fraction numerators are multiplied by their suffix multiplier before being divided by their denominator, with floating point math starting where the first float is present + + :returns: Optional[Duration] - Return value, None if it could not be parsed + """ + + # Check if in supertable + try: + return cls.from_seconds(_super_table[s]) + except KeyError: + pass + + # Quick reject anything not in allowed set + if not all(ch in _allowed_chars for ch in s): + return None + + tree = _str_to_tree(s) + + if tree is None: return None + + out = 0 + + for i in tree: + try: + v: Optional[int] + if (v := i.value(_suffix_table)) is not None: + out += v + else: + return None + except ValueError: + return None + + return cls.from_seconds(out) + + +class Time: + """ + This libraries basic representation of a point in Time, contains methods to parse and manipulate itself + """ + __slots__ = "_unix", "_monotonic", "_tz", "__dt_ptr" + + def __init__(self, *, unix: Optional[int] = None, tz: Optional[datetime.tzinfo] = None) -> None: + """ + Default constructor, generates a Time from unix seconds and a timezone + """ + + # None tz == UTC + self._tz: Optional[datetime.tzinfo] = tz + + # unix is in nanoseconds + self._unix: int + # monotonic is in nanoseconds and optionally set + self._monotonic: Optional[int] + + if unix is None: + self._unix = time.time_ns() + self._monotonic = time.monotonic_ns() + else: + self._unix = unix * _NANOS_PER_SECOND + self._monotonic = None + + # _unix and _tz must be frozen after __dt_ptr is set to preserve datetime accuracy, so disallow _unix and _tz editing + self.__dt_ptr: Optional[datetime.datetime] = None + + def __format__(self, format_str: str) -> str: + """ + Formats time using datetime strftime syntax + """ + return self.as_datetime().strftime(format_str) + + def __eq__(self, other: object) -> bool: + """ + Returns whether 2 time instants have the same time and timezone + """ + if isinstance(other, Time): + return self._unix == other._unix and self._tz == other._tz + else: + return False + + def __ne__(self, other: object) -> bool: + return not self == other + + def __le__(self, other: "Time") -> bool: + return self._unix <= other._unix + + def __lt__(self, other: "Time") -> bool: + return self._unix < other._unix + + def __ge__(self, other: "Time") -> bool: + return self._unix >= other._unix + + def __gt__(self, other: "Time") -> bool: + return self._unix > other._unix + + def __sub__(self, other: "Time") -> Duration: + return self.difference(other) + + def __add__(self, other: Duration) -> "Time": + """ + Adds a Duration to a Time. + + Note that this does not account for irregularities with what we perceive as days/months + (ex leap seconds, months have different amount of days) + for those purposes, use add_(days|months|years) + """ + + new_unix = self._unix + other.nanos() + return self.from_nanos(new_unix, tz=self._tz) + + def add_days(self, days: int) -> "Time": + """ + Adds days to the time and returns a new Time object with the new time + """ + + dt_ptr = self.as_datetime() + return self.from_datetime(dt_ptr + datetime.timedelta(days=days)) + + def add_months(self, months: int) -> "Time": + """ + Adds months to the time and returns a new Time object with the new time + """ + + dt_ptr = self.as_datetime() + + extra_years, new_months = divmod(months + dt_ptr.month, 12) + # account for divmod returning range(0..12) instead of inclusive range(1..=12) + new_months += 1 + + return self.from_datetime(dt_ptr.replace(month=new_months, year=dt_ptr.year + extra_years)) + + def add_years(self, years: int) -> "Time": + """ + Adds years to the time and returns a new Time object with the new time + """ + dt_ptr = self.as_datetime() + + return self.from_datetime(dt_ptr.replace(year=dt_ptr.year + years)) + + @classmethod + def now(cls) -> "Time": + """ + Returns a UTC timezone current time object + """ + return cls() + + @classmethod + def from_nanos(cls, nanos: int, *, tz: Optional[datetime.tzinfo] = None) -> "Time": + """ + Returns a new Time object from unix nanoseconds + """ + # disable monotonic + t = cls(unix=0, tz=tz) + t._unix = nanos + return t + + def has_monotonic(self) -> bool: + """ + Reports whether this Time object contains a monotonic clock reading + """ + return self._monotonic is not None + + def difference(self, other: "Time") -> Duration: + """ + Calculates the difference between self and other and returns a Duration representing self-other + Defaults to monotonic clock, fallsback to unix if either does not contain monotonic + """ + + if self._monotonic is not None and other._monotonic is not None: + return Duration(self._monotonic - other._monotonic) + else: + return Duration(self._unix - other._unix) + + def elapsed(self) -> Duration: + """ + Calculates the difference between current time and this Time object, + Uses monotonic clock if object has monotonic, else fallsback to unix comparison + """ + return self.now().difference(self) + + def clock(self) -> Clock: + """ + Returns a wall clock reading of hours, minutes, and seconds in the current timezone + """ + dt = self.as_datetime() + nanos = self._unix % _NANOS_PER_SECOND + return Clock(dt.hour, dt.minute, dt.second, nanos) + + def date(self) -> Date: + """ + Returns a date reading of years, months, and days in the current timezone + """ + dt = self.as_datetime() + return Date(dt.year, dt.month, dt.day) + + def UTC(self) -> "Time": + """ + Returns a new time object with the timezone set to UTC, preserving monotonic + """ + new = self.from_nanos(self._unix) + # Preserve monotonic as timezone is not related + new._monotonic = self._monotonic + return new + + def unix(self) -> int: + """ + Returns Unix seconds + """ + return self._unix // _NANOS_PER_SECOND + + def unix_milli(self) -> int: + """ + Returns Unix milliseconds + """ + return self._unix // _NANOS_PER_MILLI + + def unix_micro(self) -> int: + """ + Returns Unix microseconds + """ + return self._unix // _NANOS_PER_MICRO + + def unix_nano(self) -> int: + """ + Returns Unix nanoseconds + """ + return self._unix + + def as_datetime(self) -> datetime.datetime: + """ + Returns an aware datetime of the current time + """ + if self.__dt_ptr is None: + dt = datetime.datetime.fromtimestamp(self.unix()).astimezone(datetime.timezone.utc) + self.__dt_ptr = dt.astimezone(self._tz if self._tz is not None else datetime.timezone.utc) + + # shallow copy ensures __dt_ptr remains immutable + return pycopy.copy(self.__dt_ptr) + + @classmethod + def from_datetime(cls, dt: datetime.datetime, /) -> "Time": + """ + Builds a Time object from a datetime object, assumes naive datetimes are Localtime + Warning: Calling this method with a naive datetime object may not return the wanted Time instant + """ + if dt.tzinfo is not None and dt.tzinfo != datetime.timezone.utc: + tz = dt.tzinfo + else: + tz = None + + nanos = int(dt.timestamp() * _NANOS_PER_SECOND) + nanos += dt.microsecond * _NANOS_PER_MICRO + + self = cls.from_nanos(nanos, tz=tz) + + # cheap clone datetime if it is aware instead of having to generate a new one later + if dt.tzinfo is not None: + self.__dt_ptr = pycopy.copy(dt) + + return self From 12b811430ad66ddcdc4ad90e9cdaeffdb11cc974 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Mon, 4 Jul 2022 15:33:02 -0700 Subject: [PATCH 68/78] partially rollout lib_datetimeplus --- cmds/cmd_bookkeeping.py | 12 ++- libs/lib_goparsers.py | 216 ++-------------------------------------- 2 files changed, 17 insertions(+), 211 deletions(-) diff --git a/cmds/cmd_bookkeeping.py b/cmds/cmd_bookkeeping.py index 0e45946..5a6a598 100644 --- a/cmds/cmd_bookkeeping.py +++ b/cmds/cmd_bookkeeping.py @@ -26,13 +26,17 @@ import lib_tparse importlib.reload(lib_tparse) +import lib_datetimeplus -from lib_loaders import load_embed_color, embed_colors, datetime_now, datetime_unix +importlib.reload(lib_datetimeplus) + +from lib_loaders import load_embed_color, embed_colors from lib_db_obfuscator import db_hlapi from lib_parsers import format_duration, paginate_noexcept from lib_sonnetconfig import REGEX_VERSION from lib_sonnetcommands import CommandCtx from lib_tparse import Parser +from lib_datetimeplus import Time import lib_constants as constants from typing import List, Tuple, Optional @@ -168,7 +172,7 @@ async def get_detailed_infraction(message: discord.Message, args: List[str], cli infraction_embed.add_field(name="Reason", value=reason) infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") - infraction_embed.timestamp = datetime_unix(int(timestamp)) + infraction_embed.timestamp = Time(unix=int(timestamp)).as_datetime() try: await message.channel.send(embed=infraction_embed) @@ -208,7 +212,7 @@ async def delete_infraction(message: discord.Message, args: List[str], client: d infraction_embed.set_footer(text=f"uid: {user_id}, unix: {timestamp}") - infraction_embed.timestamp = datetime_unix(int(timestamp)) + infraction_embed.timestamp = Time(unix=int(timestamp)).as_datetime() try: await message.channel.send(embed=infraction_embed) @@ -247,7 +251,7 @@ async def query_mutedb(message: discord.Message, args: List[str], client: discor return 0 def fmtfunc(v: Tuple[str, str, int]) -> str: - ts = "No Unmute" if v[2] == 0 else format_duration(v[2] - datetime_now().timestamp()) + ts = "No Unmute" if v[2] == 0 else format_duration(v[2] - Time.now().unix()) return (f"{v[1]}, {v[0]}, {ts}") out = paginate_noexcept(sorted(table, key=lambda i: i[2]), page, per_page, 1500, fmtfunc) diff --git a/libs/lib_goparsers.py b/libs/lib_goparsers.py index 63aae06..15325d3 100644 --- a/libs/lib_goparsers.py +++ b/libs/lib_goparsers.py @@ -15,17 +15,19 @@ import ctypes as _ctypes import subprocess as _subprocess -from dataclasses import dataclass -import string from functools import lru_cache -from typing import Optional, Dict, List, Literal, Set, Any, Union +from typing import Optional import lib_sonnetconfig importlib.reload(lib_sonnetconfig) +import lib_datetimeplus + +importlib.reload(lib_datetimeplus) from lib_sonnetconfig import GOLIB_LOAD, GOLIB_VERSION +from lib_datetimeplus import Duration as _Duration class _GoString(_ctypes.Structure): @@ -184,184 +186,6 @@ def MustParseDuration(s: str) -> int: return ret -_super_table: Dict[str, int] = { - "week": 60 * 60 * 24 * 7, - "w": 60 * 60 * 24 * 7, - "day": 60 * 60 * 24, - "d": 60 * 60 * 24, - "hour": 60 * 60, - "h": 60 * 60, - "minute": 60, - "m": 60, - "second": 1, - "s": 1, - } - -_suffix_table: Dict[str, int] = dict([("weeks", 60 * 60 * 24 * 7), ("days", 60 * 60 * 24), ("hours", 60 * 60), ("minutes", 60), ("seconds", 1)] + list(_super_table.items())) - -_alpha_chars = set(char for word in _suffix_table for char in word) -_digit_chars = set(string.digits + "./") -_allowed_chars: Set[str] = _alpha_chars.union(_digit_chars) - - -@dataclass -class _Fraction: - __slots__ = "numerator", "denominator" - numerator: float - denominator: float - - -@dataclass -class _SuffixedNumber: - __slots__ = "number", "suffix" - number: Union[float, _Fraction] - suffix: str - - def value(self, multiplier_table: Dict[str, int]) -> Optional[int]: - - multiplier = multiplier_table.get(self.suffix) - - if multiplier is None: - return None - - if isinstance(self.number, (float, int)): - try: - return int(multiplier * self.number) - except ValueError: - return None - - else: - try: - return int((multiplier * self.number.numerator) // self.number.denominator) - except (ValueError, ZeroDivisionError): - return None - - -@dataclass -class _TypedStr: - __slots__ = "val", "typ" - val: str - typ: Literal["digit", "suffix"] - - -class _idx_ptr: - __slots__ = "idx", - - def __init__(self, idx: int): - self.idx = idx - - def inc(self) -> "_idx_ptr": - self.idx += 1 - return self - - def __enter__(self) -> "_idx_ptr": - return self - - def __exit__(self, *ignore: Any) -> None: - self.idx += 1 - - -def _float_or_int(s: str) -> float: - """ - Raises ValueError on failure - """ - if "." in s: - f = float(s) - if f.is_integer(): - return int(f) - else: - return f - else: - return int(s) - - -def _num_from_typedstr(v: _TypedStr) -> Optional[Union[float, _Fraction]]: - if v.typ == "digit": - try: - if "/" in v.val: - if v.val.count("/") != 1: - return None - - num, denom = v.val.split("/") - - return _Fraction(_float_or_int(num), _float_or_int(denom)) - - else: - return _float_or_int(v.val) - except ValueError: - return None - else: - return None - - -def _str_to_tree(s: str) -> Optional[List[_SuffixedNumber]]: - - tree: List[_TypedStr] = [] - - idx = _idx_ptr(0) - - while idx.idx < len(s): - with idx: - if s[idx.idx] in _digit_chars: - cache = [s[idx.idx]] - while len(s) > idx.idx + 1 and s[idx.idx + 1] in _digit_chars: - cache.append(s[idx.inc().idx]) - - tree.append(_TypedStr("".join(cache), "digit")) - elif s[idx.idx] in _alpha_chars: - - cache = [s[idx.idx]] - while len(s) > idx.idx + 1 and s[idx.idx + 1] in _alpha_chars: - cache.append(s[idx.inc().idx]) - - tree.append(_TypedStr("".join(cache), "suffix")) - - else: - return None - - if not tree: - return None - - if len(tree) == 1: - if tree[0].typ == "digit" and (n := _num_from_typedstr(tree[0])) is not None: - # disallow fractions as non prefixed numbers - if not isinstance(n, _Fraction): - return [_SuffixedNumber(n, "s")] - return None - else: - return None - - # assert len(tree) > 1 - - # Can't start on a suffix - if tree[0].typ == "suffix": - return None - - if tree[-1].typ == "digit": - return None - - # Assert that lengths are correct, should be asserted by previous logic - if not len(tree) % 2 == 0: - return None - - out: List[_SuffixedNumber] = [] - - # alternating digit and suffix starting with digit and ending on suffix - for i in range(0, len(tree), 2): - - digit = _num_from_typedstr(tree[i]) - - if digit is None: - return None - - if tree[i + 1].val not in _suffix_table: - return None - - out.append(_SuffixedNumber(digit, tree[i + 1].val)) - - return out - - @lru_cache(maxsize=500) def ParseDurationSuper(s: str) -> Optional[int]: """ @@ -377,30 +201,8 @@ def ParseDurationSuper(s: str) -> Optional[int]: :returns: Optional[int] - Return value, None if it could not be parsed """ - # Check if in supertable - try: - return _super_table[s] - except KeyError: - pass - - # Quick reject anything not in allowed set - if not all(ch in _allowed_chars for ch in s): + # Call into Duration.parse, replaces codebase of ParseDurationSuper + if (v := _Duration.parse(s)) is not None: + return v.seconds() + else: return None - - tree = _str_to_tree(s) - - if tree is None: return None - - out = 0 - - for i in tree: - try: - v: Optional[int] - if (v := i.value(_suffix_table)) is not None: - out += v - else: - return None - except ValueError: - return None - - return out From 588fbf2910f43128e61c53bec8e163687fc79c0b Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 02:42:28 -0700 Subject: [PATCH 69/78] change format style of datetime formatters, and increase datetimeplus rollout --- cmds/cmd_utils.py | 17 +++++++++++------ cmds/cmd_version.py | 16 ++++++++-------- dlibs/dlib_userupdate.py | 3 +-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index 0774524..c1a11b9 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -36,6 +36,9 @@ import lib_tparse importlib.reload(lib_tparse) +import lib_datetimeplus + +importlib.reload(lib_datetimeplus) from typing import Any, Final, List, Optional, Tuple, Dict, cast @@ -43,11 +46,12 @@ import lib_lexdpyk_h as lexdpyk from lib_compatibility import discord_datetime_now, user_avatar_url from lib_db_obfuscator import db_hlapi -from lib_loaders import datetime_now, embed_colors, load_embed_color +from lib_loaders import embed_colors, load_embed_color from lib_parsers import (parse_boolean, parse_permissions, parse_core_permissions, parse_user_member_noexcept, parse_channel_message_noexcept, generate_reply_field, grab_files) from lib_sonnetcommands import CallCtx, CommandCtx, SonnetCommand from lib_sonnetconfig import BOT_NAME from lib_tparse import Parser +from lib_datetimeplus import Time def add_timestamp(embed: discord.Embed, name: str, start: int, end: int) -> None: @@ -80,9 +84,10 @@ async def ping_function(message: discord.Message, args: List[str], client: disco await sent_message.edit(embed=ping_embed) +# Must use datetime due to discord.py being naive def parsedate(indata: Optional[datetime]) -> str: if indata is not None: - basetime = time.strftime('%a, %d %b %Y %H:%M:%S', indata.utctimetuple()) + basetime = format(indata, ':%a, %d %b %Y %H:%M:%S') days = (discord_datetime_now() - indata).days return f"{basetime} ({days} day{'s' * (days != 1)} ago)" else: @@ -138,7 +143,7 @@ async def profile_function(message: discord.Message, args: List[str], client: di if moderator or (viewinfs and user.id == message.author.id): embed.add_field(name="Infractions", value=f"{db.grab_filter_infractions(user=user.id, count=True)}") - embed.timestamp = datetime_now() + embed.timestamp = Time.now().as_datetime() try: await message.channel.send(embed=embed) except discord.errors.Forbidden: @@ -153,7 +158,7 @@ async def avatar_function(message: discord.Message, args: List[str], client: dis embed = discord.Embed(description=f"{user.mention}'s Avatar", color=load_embed_color(message.guild, embed_colors.primary, kwargs["ramfs"])) embed.set_image(url=user_avatar_url(user)) - embed.timestamp = datetime_now() + embed.timestamp = Time.now().as_datetime() try: await message.channel.send(embed=embed) except discord.errors.Forbidden: @@ -437,7 +442,7 @@ async def grab_role_info(message: discord.Message, args: List[str], client: disc r_embed.add_field(name=f"Permissions ({role.permissions.value})", value=perms_to_str(role.permissions)) r_embed.set_footer(text=f"id: {role.id}") - r_embed.timestamp = datetime_now() + r_embed.timestamp = Time.now().as_datetime() try: await message.channel.send(embed=r_embed) @@ -493,7 +498,7 @@ async def grab_guild_message(message: discord.Message, args: List[str], client: if sendraw: file_content = io.BytesIO(discord_message.content.encode("utf8")) - fileobjs.append(discord.File(file_content, filename=f"{discord_message.id}.at.{int(datetime_now().timestamp())}.txt")) + fileobjs.append(discord.File(file_content, filename=f"{discord_message.id}.at.{Time.now().unix()}.txt")) try: await message.channel.send(embed=message_embed, files=fileobjs) diff --git a/cmds/cmd_version.py b/cmds/cmd_version.py index 57254b7..4c595c6 100644 --- a/cmds/cmd_version.py +++ b/cmds/cmd_version.py @@ -5,7 +5,6 @@ import discord import io -import time import platform import lib_loaders @@ -17,8 +16,12 @@ import lib_lexdpyk_h importlib.reload(lib_lexdpyk_h) +import lib_datetimeplus -from lib_loaders import clib_exists, DotHeaders, datetime_unix +importlib.reload(lib_datetimeplus) + +from lib_loaders import clib_exists, DotHeaders +from lib_datetimeplus import Time from typing import List, Any, Union import lib_lexdpyk_h as lexdpyk @@ -49,14 +52,11 @@ def getdelta(past: Union[int, float]) -> str: Formats a delta between a past time and now to be human readable """ - trunning = (datetime_unix(int(time.time())) - datetime_unix(int(past))) + clock = (Time.now() - Time(unix=int(past))).clock() - seconds = trunning.seconds % 60 - minutes = ((trunning.seconds) // 60 % 60) - hours = ((trunning.seconds) // (60 * 60)) - days = trunning.days + days, hours = divmod(clock.hours, 24) - hms = f"{hours:02d}:{minutes:02d}:{seconds:02d}" + hms = f"{hours:02d}:{clock.minutes:02d}:{clock.seconds:02d}" # if days is 0 don't bother rendering it if days == 0: return hms diff --git a/dlibs/dlib_userupdate.py b/dlibs/dlib_userupdate.py index b9c793a..443b9f4 100644 --- a/dlibs/dlib_userupdate.py +++ b/dlibs/dlib_userupdate.py @@ -3,7 +3,6 @@ import asyncio import importlib -import time from datetime import datetime import discord @@ -61,7 +60,7 @@ async def on_member_update(before: discord.Member, after: discord.Member, **karg def parsedate(indata: Optional[datetime]) -> str: if indata is not None: - basetime = time.strftime('%a, %d %b %Y %H:%M:%S', indata.utctimetuple()) + basetime = format(indata, '%a, %d %b %Y %H:%M:%S') days = (discord_datetime_now() - indata).days return f"{basetime} ({days} day{'s' * (days != 1)} ago)" else: From 5d81380d1172b4d4cf39b702d7717648a72bb4c0 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 02:55:13 -0700 Subject: [PATCH 70/78] add in_timezone method to Time --- libs/lib_datetimeplus.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libs/lib_datetimeplus.py b/libs/lib_datetimeplus.py index 7b71543..196d052 100644 --- a/libs/lib_datetimeplus.py +++ b/libs/lib_datetimeplus.py @@ -388,9 +388,9 @@ def __init__(self, *, unix: Optional[int] = None, tz: Optional[datetime.tzinfo] def __format__(self, format_str: str) -> str: """ - Formats time using datetime strftime syntax + Formats time using datetime supported syntax """ - return self.as_datetime().strftime(format_str) + return format(self.as_datetime(), format_str) def __eq__(self, other: object) -> bool: """ @@ -431,6 +431,17 @@ def __add__(self, other: Duration) -> "Time": new_unix = self._unix + other.nanos() return self.from_nanos(new_unix, tz=self._tz) + def in_timezone(self, tz: datetime.tzinfo) -> "Time": + """ + Returns a new Time object with the timezone set to the new timezone specified. + + Preserves monotonic clock, as this operation does not change the time itself + """ + copy = pycopy.copy(self) + copy.__dt_ptr = None + copy._tz = tz + return copy + def add_days(self, days: int) -> "Time": """ Adds days to the time and returns a new Time object with the new time From 6d1246ff6e99ea920f2542c1e82558ca2b366f19 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 03:11:38 -0700 Subject: [PATCH 71/78] add repr and .local methods to Time --- libs/lib_datetimeplus.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libs/lib_datetimeplus.py b/libs/lib_datetimeplus.py index 196d052..fc58d96 100644 --- a/libs/lib_datetimeplus.py +++ b/libs/lib_datetimeplus.py @@ -22,7 +22,7 @@ _NANOS_PER_MICRO: Final = 1000 # parse method data -_super_table: Dict[str, int] = { +_super_table: Final[Dict[str, int]] = { "week": 60 * 60 * 24 * 7, "w": 60 * 60 * 24 * 7, "day": 60 * 60 * 24, @@ -386,6 +386,12 @@ def __init__(self, *, unix: Optional[int] = None, tz: Optional[datetime.tzinfo] # _unix and _tz must be frozen after __dt_ptr is set to preserve datetime accuracy, so disallow _unix and _tz editing self.__dt_ptr: Optional[datetime.datetime] = None + def __repr__(self) -> str: + + tz = self._tz if self._tz is not None else datetime.timezone.utc + + return f"Time(time={self.as_datetime().isoformat('T')}, tz={tz})" + def __format__(self, format_str: str) -> str: """ Formats time using datetime supported syntax @@ -431,6 +437,12 @@ def __add__(self, other: Duration) -> "Time": new_unix = self._unix + other.nanos() return self.from_nanos(new_unix, tz=self._tz) + def local(self) -> "Time": + """ + Returns an instance of Time with the timezone changed to the local timezone set by the system + """ + return self.from_datetime(self.as_datetime().astimezone()) + def in_timezone(self, tz: datetime.tzinfo) -> "Time": """ Returns a new Time object with the timezone set to the new timezone specified. From 59572dc677adb3c519c0b50aca136883f5c01e54 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:13:23 -0700 Subject: [PATCH 72/78] add timezone and from_date methods, patch from_datetime being bugged on microseconds, add Time.LOCAL_TZ class attr --- libs/lib_datetimeplus.py | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/libs/lib_datetimeplus.py b/libs/lib_datetimeplus.py index fc58d96..19eb248 100644 --- a/libs/lib_datetimeplus.py +++ b/libs/lib_datetimeplus.py @@ -8,7 +8,7 @@ from dataclasses import dataclass import string -from typing import Optional, Final, List, Any, Union, Literal, Set, Dict +from typing import Optional, Final, List, Any, Union, Literal, Set, Dict, TypeVar __all__ = [ "Time", @@ -21,6 +21,21 @@ _NANOS_PER_MILLI: Final = 1000 * 1000 _NANOS_PER_MICRO: Final = 1000 +_T = TypeVar("_T") + + +def _unwrap(v: Optional[_T]) -> _T: + if v is None: + raise RuntimeError("Unwrapped value was None variant") + return v + + +# I don't want to talk about it +# basically calling astimezone with no arguments will make a datetime use local time, and then grabbing tzinfo off of that gives us a local timezone +# cursed +# awful +_LOCAL_TIMEZONE = _unwrap(datetime.datetime.now(datetime.timezone.utc).astimezone().tzinfo) + # parse method data _super_table: Final[Dict[str, int]] = { "week": 60 * 60 * 24 * 7, @@ -244,6 +259,12 @@ class Date: class Duration(int): + def __repr__(self) -> str: + + c = self.clock() + + return f"Duration({c.hours}:{c.minutes}:{c.seconds}.{c.nanoseconds:09})" + def nanos(self) -> int: """ Returns duration as nanoseconds @@ -363,6 +384,8 @@ class Time: """ __slots__ = "_unix", "_monotonic", "_tz", "__dt_ptr" + LOCAL_TZ: datetime.tzinfo = _LOCAL_TIMEZONE + def __init__(self, *, unix: Optional[int] = None, tz: Optional[datetime.tzinfo] = None) -> None: """ Default constructor, generates a Time from unix seconds and a timezone @@ -439,9 +462,11 @@ def __add__(self, other: Duration) -> "Time": def local(self) -> "Time": """ - Returns an instance of Time with the timezone changed to the local timezone set by the system + Returns an instance of Time with the timezone changed to the local timezone set by the system. + + Preserves monotonic clock, as this operation does not change the time itself """ - return self.from_datetime(self.as_datetime().astimezone()) + return self.in_timezone(self.LOCAL_TZ) def in_timezone(self, tz: datetime.tzinfo) -> "Time": """ @@ -454,6 +479,12 @@ def in_timezone(self, tz: datetime.tzinfo) -> "Time": copy._tz = tz return copy + def timezone(self) -> datetime.tzinfo: + """ + Returns the timezone this Time object contains + """ + return self._tz if self._tz is not None else datetime.timezone.utc + def add_days(self, days: int) -> "Time": """ Adds days to the time and returns a new Time object with the new time @@ -578,6 +609,10 @@ def as_datetime(self) -> datetime.datetime: """ if self.__dt_ptr is None: dt = datetime.datetime.fromtimestamp(self.unix()).astimezone(datetime.timezone.utc) + + # add microseconds accuracy + dt = dt + datetime.timedelta(microseconds=(self.unix_micro() % (1000 * 1000))) + self.__dt_ptr = dt.astimezone(self._tz if self._tz is not None else datetime.timezone.utc) # shallow copy ensures __dt_ptr remains immutable @@ -594,7 +629,7 @@ def from_datetime(cls, dt: datetime.datetime, /) -> "Time": else: tz = None - nanos = int(dt.timestamp() * _NANOS_PER_SECOND) + nanos = int(dt.timestamp()) * _NANOS_PER_SECOND nanos += dt.microsecond * _NANOS_PER_MICRO self = cls.from_nanos(nanos, tz=tz) @@ -604,3 +639,11 @@ def from_datetime(cls, dt: datetime.datetime, /) -> "Time": self.__dt_ptr = pycopy.copy(dt) return self + + @classmethod + def from_date(cls, date: Date, /, *, tz: Optional[datetime.tzinfo] = None) -> "Time": + + if tz is None: + tz = datetime.timezone.utc + + return cls.from_datetime(datetime.datetime(date.years, date.months, date.days).astimezone(tz)) From 3635426a3c115ce9b50164240261ea283da56b9c Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:29:41 -0700 Subject: [PATCH 73/78] remove -3.9 restriction, sonnet works on 3.10 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9725807..fccee4f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Stable](https://github.com/Sonnet-Discord/sonnet-py/actions/workflows/python-package.yml/badge.svg?branch=main)](https://github.com/Sonnet-Discord/sonnet-py/actions/workflows/python-package.yml) [![Dev branch](https://github.com/Sonnet-Discord/sonnet-py/actions/workflows/python-dev.yml/badge.svg?branch=dev-unstable)](https://github.com/Sonnet-Discord/sonnet-py/actions/workflows/python-dev.yml) # Sonnet py -A discord bot written in python 3.8-3.9 with a focus on moderation +A discord bot written in python 3.8 with a focus on moderation More details are available at [The Sonnet Website](https://sonnet-discord.github.io) From fcc6dec3c0cfeed87df640c77017a1f5a891e770 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:37:48 -0700 Subject: [PATCH 74/78] swap workflows to use ubuntu 22.04 (20.04 has an outdated dependency) --- .github/workflows/python-dev.yml | 2 +- .github/workflows/python-package.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index f7b73b6..a5c41a3 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -12,7 +12,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python-version: [3.8] diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 23fa558..f73cc31 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python-version: [3.8] From 5c3ee1e06b876ec8a501af6f0a1e492c4a7b506a Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:46:00 -0700 Subject: [PATCH 75/78] revert 22.04 and lock to 20.04, sonnet should run on 20.04 --- .github/workflows/python-dev.yml | 2 +- .github/workflows/python-package.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index a5c41a3..072dbfc 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -12,7 +12,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 strategy: matrix: python-version: [3.8] diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f73cc31..4208fc4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-20.04 strategy: matrix: python-version: [3.8] @@ -57,7 +57,7 @@ jobs: run: | sudo apt update sudo apt install libmariadb-dev-compat - python -m pip install mariadb + python -m pip install mariadb>=1.0.11,<2 - name: run pyright type checking run: | pyright From 52d0a6ac5fe5150783afd8226e962872c93ef109 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:47:37 -0700 Subject: [PATCH 76/78] fix for fact that this is in a shell --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 4208fc4..8d0abf3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,7 @@ jobs: run: | sudo apt update sudo apt install libmariadb-dev-compat - python -m pip install mariadb>=1.0.11,<2 + python -m pip install "mariadb>=1.0.11,<2" - name: run pyright type checking run: | pyright From f413e9823be947f3416a67885c43215f674d6d46 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 16:50:20 -0700 Subject: [PATCH 77/78] lock build tests to old mariadb version --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8d0abf3..aac2b44 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,7 @@ jobs: run: | sudo apt update sudo apt install libmariadb-dev-compat - python -m pip install "mariadb>=1.0.11,<2" + python -m pip install "mariadb>=1.0.11,<1.1" - name: run pyright type checking run: | pyright From 89e7b7b783f693cf2876c51803626084aeb24b65 Mon Sep 17 00:00:00 2001 From: ultrabear Date: Tue, 5 Jul 2022 22:11:12 -0700 Subject: [PATCH 78/78] move versions to rc status --- cmds/cmd_administration.py | 2 +- cmds/cmd_automod.py | 2 +- cmds/cmd_bookkeeping.py | 2 +- cmds/cmd_moderation.py | 2 +- cmds/cmd_scripting.py | 2 +- cmds/cmd_utils.py | 2 +- cmds/cmd_version.py | 2 +- dlibs/dlib_messages.py | 2 +- dlibs/dlib_userupdate.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmds/cmd_administration.py b/cmds/cmd_administration.py index 93bab53..a1d9753 100644 --- a/cmds/cmd_administration.py +++ b/cmds/cmd_administration.py @@ -510,4 +510,4 @@ async def set_filelog_behavior(message: discord.Message, args: List[str], client } } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_automod.py b/cmds/cmd_automod.py index 29dee62..40a090d 100644 --- a/cmds/cmd_automod.py +++ b/cmds/cmd_automod.py @@ -690,4 +690,4 @@ async def add_joinrule(message: discord.Message, args: List[str], client: discor }, } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_bookkeeping.py b/cmds/cmd_bookkeeping.py index 5a6a598..cec5a61 100644 --- a/cmds/cmd_bookkeeping.py +++ b/cmds/cmd_bookkeeping.py @@ -344,4 +344,4 @@ async def remove_mutedb(message: discord.Message, args: List[str], client: disco }, } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_moderation.py b/cmds/cmd_moderation.py index 01bb9ee..0e897c2 100644 --- a/cmds/cmd_moderation.py +++ b/cmds/cmd_moderation.py @@ -691,4 +691,4 @@ async def purge_cli(message: discord.Message, args: List[str], client: discord.C } } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_scripting.py b/cmds/cmd_scripting.py index c2500f8..c2d3f42 100644 --- a/cmds/cmd_scripting.py +++ b/cmds/cmd_scripting.py @@ -430,4 +430,4 @@ async def sleep_for(message: discord.Message, args: List[str], client: discord.C }, } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_utils.py b/cmds/cmd_utils.py index c1a11b9..34f1fcc 100644 --- a/cmds/cmd_utils.py +++ b/cmds/cmd_utils.py @@ -619,4 +619,4 @@ async def coinflip(message: discord.Message, args: List[str], client: discord.Cl } } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/cmds/cmd_version.py b/cmds/cmd_version.py index 4c595c6..56ea4d3 100644 --- a/cmds/cmd_version.py +++ b/cmds/cmd_version.py @@ -184,4 +184,4 @@ async def print_stats(message: discord.Message, args: List[str], client: discord } } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13" diff --git a/dlibs/dlib_messages.py b/dlibs/dlib_messages.py index de3031d..2dd038f 100644 --- a/dlibs/dlib_messages.py +++ b/dlibs/dlib_messages.py @@ -559,4 +559,4 @@ async def on_message(message: discord.Message, kernel_args: lexdpyk.KernelArgs) "on-message-delete": on_message_delete, } -version_info: Final = "1.2.13-DEV" +version_info: Final = "1.2.13" diff --git a/dlibs/dlib_userupdate.py b/dlibs/dlib_userupdate.py index 443b9f4..c09327d 100644 --- a/dlibs/dlib_userupdate.py +++ b/dlibs/dlib_userupdate.py @@ -195,4 +195,4 @@ async def on_member_remove(member: discord.Member, **kargs: Any) -> None: "on-member-remove": on_member_remove, } -version_info: str = "1.2.13-DEV" +version_info: str = "1.2.13"