Add Matrix support

Author: CaptainIRS

.env.example

@@ -1,28 +1,43 @@
 DISCORD_TOKEN=
 
-DISCORD_FUN_REDDIT=
-DISCORD_SERIOUS_REDDIT=
-DISCORD_LIFE_REDDIT=
-DISCORD_COMPANY_BLOGS=
-DISCORD_PRODUCT_BLOGS=
-DISCORD_INFOSEC_BLOGS=
-DISCORD_INDIVIDUAL_BLOGS=
-DISCORD_NEWS=
-DISCORD_FORUMS=
-DISCORD_DEBUG_CHANNEL=
+DISCORD_FUN_REDDIT=0
+DISCORD_SERIOUS_REDDIT=0
+DISCORD_LIFE_REDDIT=0
+DISCORD_COMPANY_BLOGS=0
+DISCORD_PRODUCT_BLOGS=0
+DISCORD_INFOSEC_BLOGS=0
+DISCORD_INDIVIDUAL_BLOGS=0
+DISCORD_NEWS=0
+DISCORD_FORUMS=0
+DISCORD_DEBUG_CHANNEL=0
 
 TELEGRAM_TOKEN=
 TELEGRAM_CHAT_ID=
 
-TELEGRAM_FUN_REDDIT=
-TELEGRAM_SERIOUS_REDDIT=
-TELEGRAM_LIFE_REDDIT=
-TELEGRAM_COMPANY_BLOGS=
-TELEGRAM_PRODUCT_BLOGS=
-TELEGRAM_INDIVIDUAL_BLOGS=
-TELEGRAM_INFOSEC_BLOGS=
-TELEGRAM_FORUMS=
-TELEGRAM_NEWS=
+TELEGRAM_FUN_REDDIT=0
+TELEGRAM_SERIOUS_REDDIT=0
+TELEGRAM_LIFE_REDDIT=0
+TELEGRAM_COMPANY_BLOGS=0
+TELEGRAM_PRODUCT_BLOGS=0
+TELEGRAM_INDIVIDUAL_BLOGS=0
+TELEGRAM_INFOSEC_BLOGS=0
+TELEGRAM_FORUMS=0
+TELEGRAM_NEWS=0
+
+#matrix
+MATRIX_HOMESERVER=
+MATRIX_USER=
+MATRIX_TOKEN=
+
+MATRIX_FUN_REDDIT=
+MATRIX_SERIOUS_REDDIT=
+MATRIX_LIFE_REDDIT=
+MATRIX_COMPANY_BLOGS=
+MATRIX_PRODUCT_BLOGS=
+MATRIX_INDIVIDUAL_BLOGS=
+MATRIX_INFOSEC_BLOGS=
+MATRIX_FORUMS=
+MATRIX_NEWS=
 
 #reddit auth
 CLIENT_ID=

data/blogs/company_blogs.opml

@@ -117,6 +117,7 @@
       <outline type="rss" text="Intel" title="Intel" xmlUrl="https://community.intel.com/cipcp26785/rss/Category?category.id=tech-innovation&amp;interaction.style=blog" htmlUrl="https://community.intel.com/t5/Tech-Innovation/ct-p/tech-innovation"/>
       <outline type="rss" text="Intercom" title="Intercom" xmlUrl="https://www.intercom.com/blog/engineering/feed" htmlUrl="https://engineering.intercom.io/"/>
       <outline type="rss" text="Jane Street" title="Jane Street" xmlUrl="https://blogs.janestreet.com/feed.xml" htmlUrl="https://blogs.janestreet.com/category/ocaml/"/>
+      <outline type="rss" text="JioCinema" title="JioCinema" xmlUrl="https://blog.jiocinema.com/rss/" htmlUrl="https://blog.jiocinema.com/"/>
       <outline type="rss" text="Jobandtalent" title="Jobandtalent" xmlUrl="https://jobandtalent.engineering/feed" htmlUrl="https://jobandtalent.engineering/"/>
       <outline type="rss" text="Just Eat" title="Just Eat" xmlUrl="https://tech.just-eat.com/feed/" htmlUrl="https://tech.just-eat.com/"/>
       <outline type="rss" text="King" title="King" xmlUrl="https://techblog.king.com/feed/" htmlUrl="https://techblog.king.com/"/>

data/blogs/individual_blogs.opml

@@ -178,6 +178,7 @@
         <outline title="mfkl" htmlUrl="https://mfkl.github.io" xmlUrl="https://mfkl.github.io/feed.xml" />
         <outline title="mgsloan" htmlUrl="https://mgsloan.com" xmlUrl="https://mgsloan.com/feed.xml" />
         <outline title="Michael Chadwick | home of audio dev michael chadwick" htmlUrl="https://michaelchadwick.info" xmlUrl="https://michaelchadwick.info/feed.xml" />
+        <outline title="mcyoung" htmlUrl="https://mcyoung.xyz/" xmlUrl="https://mcyoung.xyz/feed.xml" />
         <outline title="Miika von Bell: Full-stack software developer" htmlUrl="https://miikavonbell.com" xmlUrl="https://miikavonbell.com/index.xml" />
         <outline title="www.mikenikles.com" htmlUrl="https://mikenikles.com" xmlUrl="https://www.mikenikles.com/rss.xml" />
         <outline title="Mazzarolo Matteo · mmazzarolo.com" htmlUrl="https://mmazzarolo.com" xmlUrl="https://mmazzarolo.com/blog/index.xml" />
@@ -333,4 +334,4 @@
         <outline type="rss" title="Jezen Thomas | Haskell, Unix, Minimalism, and Entrepreneurship." xmlUrl="https://jezenthomas.com/feed.xml" htmlUrl="https://jezenthomas.com" />
     </outline>
     </body>
-</opml>
+</opml>

data/blogs/news.opml

@@ -10,7 +10,6 @@
         <outline text="Coin Desk" title="Coin Desk" type="rss" xmlUrl="https://www.coindesk.com/arc/outboundfeeds/rss/" htmlUrl="https://www.coindesk.com/"/>
         <outline text="Computer Weekly" title="Computer Weekly" type="rss" xmlUrl="https://www.computerweekly.com/rss/RSS-Feed.xml" htmlUrl="https://www.computerweekly.com/"/>
         <outline text="Computer World" title="Computer World" type="rss" xmlUrl="https://www.computerworld.com/in/index.rss" htmlUrl="https://www.computerworld.com/"/>
-        <outline text="Extreme Tech" title="Extreme Tech" type="rss" xmlUrl="https://www.extremetech.com/feed" htmlUrl="https://www.extremetech.com/"/>
         <outline text="Fast Company" title="Fast Company" type="rss" xmlUrl="https://www.fastcompany.com/technology/rss" htmlUrl="https://www.fastcompany.com/technology"/>
         <outline text="Information Week" title="Information Week" type="rss" xmlUrl="https://www.informationweek.com/rss.xml" htmlUrl="https://www.informationweek.com/"/>
         <outline text="MIT Technology Review" title="MIT Technology Review" type="rss" xmlUrl="https://www.technologyreview.com/feed/" htmlUrl="https://www.technologyreview.com/"/>

main.py

@@ -10,6 +10,7 @@
 from time import sleep
 
 import discord
+import simplematrixbotlib as botlib
 import telegram
 from discord.ext import commands
 from dotenv import load_dotenv
@@ -19,10 +20,12 @@
 import forums
 import reddit
 from util.embed_utils import (RedditPost, TelegramPost, create_blog_embed,
-                              create_blog_reddit_post,
+                              create_blog_matrix_post, create_blog_reddit_post,
                               create_blog_telegram_post, create_forum_embed,
+                              create_forum_matrix_post,
                               create_forum_reddit_post,
                               create_forum_telegram_post, create_reddit_embed,
+                              create_reddit_matrix_post,
                               create_reddit_telegram_post)
 from util.load_config import load_config
 
@@ -78,6 +81,23 @@
     'forum': os.getenv('REDDIT_FORUMS'),
 }
 
+matrix_space = os.getenv('MATRIX_SPACE')
+matrix_rooms = {
+    'reddit': {
+        'fun': os.getenv('MATRIX_FUN_REDDIT'),
+        'serious': os.getenv('MATRIX_SERIOUS_REDDIT'),
+        'life': os.getenv('MATRIX_LIFE_REDDIT'),
+    },
+    'blog': {
+        'company': os.getenv('MATRIX_COMPANY_BLOGS'),
+        'product': os.getenv('MATRIX_PRODUCT_BLOGS'),
+        'infosec': os.getenv('MATRIX_INFOSEC_BLOGS'),
+        'news': os.getenv('MATRIX_NEWS'),
+        'individual': os.getenv('MATRIX_INDIVIDUAL_BLOGS'),
+    },
+    'forum': os.getenv('MATRIX_FORUMS'),
+}
+
 
 class Updater:  # pylint: disable=too-few-public-methods
     '''
@@ -137,7 +157,8 @@ class DiscordBot(commands.Bot):
 
     def __init__(self, command_prefix: str, queue: JoinableQueue):
         self.queue = queue
-        super().__init__(command_prefix=command_prefix)
+        intents = discord.Intents.default()
+        super().__init__(command_prefix=command_prefix, intents=intents)
 
     async def on_ready(self):
         '''
@@ -189,6 +210,47 @@ def run_discord_bot(queue: JoinableQueue):
     discord_bot.run(DISCORD_TOKEN)
 
 
+def run_matrix_bot(queue: JoinableQueue):
+    '''
+    Run matrix bot
+    '''
+    creds = botlib.Creds(
+        os.getenv('MATRIX_HOMESERVER'),
+        os.getenv('MATRIX_USER'),
+        access_token=os.getenv('MATRIX_TOKEN'))
+    bot = botlib.Bot(creds)
+
+    @bot.listener.on_startup
+    async def on_startup(room_id: str):
+        if room_id != matrix_space:
+            return
+        while True:
+            message = queue.get()
+            if message is None:
+                queue.task_done()
+                break
+            source, channel, post = message
+            if source == 'reddit':
+                matrix_post = create_reddit_matrix_post(channel, post)
+                await bot.api.send_markdown_message(
+                    room_id=matrix_rooms['reddit'][channel['category']],
+                    message=matrix_post.markdown)
+            elif source == 'blog':
+                matrix_post = create_blog_matrix_post(post)
+                await bot.api.send_markdown_message(
+                    room_id=matrix_rooms['blog'][channel],
+                    message=matrix_post.markdown)
+            elif source == 'forum':
+                matrix_post = create_forum_matrix_post(channel, post)
+                await bot.api.send_markdown_message(
+                    room_id=matrix_rooms['forum'],
+                    message=matrix_post.markdown)
+            queue.task_done()
+            await asyncio.sleep(1)
+        await bot.async_client.logout()
+    bot.run()
+
+
 async def send_to_telegram_channel(telegram_bot: telegram.Bot, thread_id: str, post: TelegramPost):
     '''
     Send message to telegram channel
@@ -325,6 +387,12 @@ def message_sender(message_queue: JoinableQueue):
         queues.append(reddit_queue)
         reddit_process = Process(target=run_reddit_bot, args=(reddit_queue,))
         reddit_process.start()
+    if os.getenv('MATRIX_USER') and os.getenv('MATRIX_TOKEN'):
+        logging.info('Starting matrix bot')
+        matrix_queue = JoinableQueue()
+        queues.append(matrix_queue)
+        matrix_process = Process(target=run_matrix_bot, args=(matrix_queue,))
+        matrix_process.start()
     while True:
         message = message_queue.get()
         if message is None:

requirements-dev.txt

@@ -1,16 +1,17 @@
 markdownify==0.6.1
-markdown==3.3.4
-discord.py==1.6.0
-praw==7.1.0
+markdown==3.3.7
+discord.py==2.0.0
+praw==7.7.1
 opml==0.5
 feedparser==6.0.2
 python-dotenv==0.19.1
 python-telegram-bot==20.5
+simplematrixbotlib==2.12.0
 
 # Dev dependencies
 autopep8==1.5.7
 isort==5.7.0
 pre-commit==3.4.0
 pycodestyle==2.7.0
-pylint==2.17.6
+pylint==3.0.0
 flake8==3.9.2

requirements.txt

@@ -1,8 +1,9 @@
 markdownify==0.6.1
-markdown==3.3.4
-discord.py==1.6.0
-praw==7.1.0
+markdown==3.3.7
+discord.py==2.0.0
+praw==7.7.1
 opml==0.5
 feedparser==6.0.2
 python-dotenv==0.19.1
 python-telegram-bot==20.5
+simplematrixbotlib==2.12.0

util/embed_utils.py

@@ -56,7 +56,7 @@ def set_footer(self, **kwargs):
         if icon_url == '':
             icon_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/bd/' \
                 'Ic_account_circle_48px.svg/200px-Ic_account_circle_48px.svg.png'
-        icon_url = f'https://images.weserv.nl/?url={icon_url}&output=png&bg=white&w=100&h=100&fit=contain&cbg=white'
+        icon_url = f'https: //images.weserv.nl/?url={icon_url}&output=png&bg=white&w=100&h=100&fit=contain&cbg=white'
         self.embed.set_footer(
             text=text,
             icon_url=icon_url
@@ -319,3 +319,76 @@ def create_blog_reddit_post(post):
         description=description,
         url=url
     )
+
+
+@dataclass
+class MatrixPost:
+    '''
+    Matrix post
+    '''
+    markdown: str
+
+
+def create_forum_matrix_post(forum, post):
+    '''
+    Create forum embed from JSON
+    '''
+    config = load_config('forums.json')[forum]
+
+    message = f'💬 {config["publisher"]}'
+    message += f'\n\n**[{post["title"]}]({post["url"]})**'
+    if post["description"]:
+        if len(post["description"]) > 1000:
+            post["description"] = post["description"][:1000] + '...'
+        message += f'\n\n{post["description"]}'
+    if 'comments_link' in post:
+        while '  ' in post['comments_link']:
+            post['comments_link'] = post['comments_link'].replace('  ', ' ')
+        message += f'\n\n{post["comments_link"]}'
+    message += '\n‎'
+
+    return MatrixPost(
+        markdown=message
+    )
+
+
+def create_blog_matrix_post(post):
+    '''
+    Create blog post embed from JSON
+    '''
+    message = f'📰 {post["publisher"]}'
+    message += f'\n\n**[{post["title"]}]({post["url"]})**'
+    image_url = post["image"] if 'image' in post and len(post['image']) > 0 else ''
+    if image_url:
+        message += f'\n\n![{post["title"]}]({image_url})'
+    if post["description"]:
+        if len(post["description"]) > 1000:
+            post["description"] = post["description"][:1000] + '...'
+        message += f'\n\n{post["description"].strip()}'
+    message += '\n‎'
+    while '\n\n\n' in message:
+        message = message.replace('\n\n\n', '\n\n')
+
+    return MatrixPost(
+        markdown=message
+    )
+
+
+def create_reddit_matrix_post(subreddit, post):
+    '''
+    Create reddit post embed from subreddit JSON
+    '''
+    message = f'💬 r/{subreddit["subreddit"]}'
+    message += f'\n\n**[{post["title"]}]({post["url"]})**'
+    image_url = post["media"]["url"] if 'media' in post and 'url' in post['media'] else ''
+    if image_url:
+        message += f'\n\n![{post["title"]}]({image_url})'
+    if post["selftext"]:
+        if len(post["selftext"]) > 1000:
+            post["selftext"] = post["selftext"][:1000] + '...'
+        message += f'\n\n{post["selftext"].strip()}'
+    message += '\n‎'
+
+    return MatrixPost(
+        markdown=message
+    )