From de883943c6186f36d84d4f84a0d5b8be924a3049 Mon Sep 17 00:00:00 2001 From: proffapt Date: Thu, 25 Jul 2024 17:07:46 +0530 Subject: [PATCH] feat: filtered subscribers --- .gitignore | 1 + docker-compose.yml | 52 ++++++++++---------- mftp/.dockerignore | 3 +- mftp/README.md | 4 ++ mftp/docker-compose.yml | 35 +++++++------- mftp/env.example.py | 21 ++++++-- mftp/notice.py | 53 ++++++++++++++++++++ mftp/ntfy.py | 104 ++++++++++++++++++++++++---------------- 8 files changed, 182 insertions(+), 91 deletions(-) diff --git a/.gitignore b/.gitignore index 21fa803..6260828 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ env.py # logs logs.txt .lsnif +.ntfy.lsnsf # configs mftp-service-aliases diff --git a/docker-compose.yml b/docker-compose.yml index edbe9f0..c7198b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,30 @@ version: "3" services: + mftp: + build: + context: ./mftp + dockerfile: Dockerfile + image: metakgporg/mftp + container_name: mftp + command: $MFTP_MODE + volumes: + - $MFTP_CONFIG/env.py:/app/env.py + - $MFTP_CONFIG/token.json:/app/token.json + - $MFTP_CONFIG/credentials.json:/app/credentials.json + - $MFTP_CONFIG/mail_send_token.json:/app/mail_send_token.json + - $MFTP_CONFIG/mail_send_creds.json:/app/mail_send_creds.json + - $MFTP_CONFIG/.lsnif:/app/.lsnif + - $MFTP_CONFIG/.ntfy.lsnsf:/app/.ntfy.lsnsf + - $MFTP_CONFIG/.session:/app/.session - mftp: - build: - context: ./mftp - dockerfile: Dockerfile - image: metakgporg/mftp - container_name: mftp - command: $MFTP_MODE - volumes: - - $MFTP_CONFIG/env.py:/app/env.py - - $MFTP_CONFIG/token.json:/app/token.json - - $MFTP_CONFIG/credentials.json:/app/credentials.json - - $MFTP_CONFIG/mail_send_token.json:/app/mail_send_token.json - - $MFTP_CONFIG/mail_send_creds.json:/app/mail_send_creds.json - - $MFTP_CONFIG/.lsnif:/app/.lsnif - - $MFTP_CONFIG/.session:/app/.session - - mftp-doctor: - build: - context: ./mftp-doctor - dockerfile: Dockerfile - image: metakgporg/mftp-doctor - container_name: mftp-doctor - command: ${DOCTOR_MODE:-} - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - $DOCTOR_CONFIG/env.py:/app/env.py + mftp-doctor: + build: + context: ./mftp-doctor + dockerfile: Dockerfile + image: metakgporg/mftp-doctor + container_name: mftp-doctor + command: ${DOCTOR_MODE:-} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - $DOCTOR_CONFIG/env.py:/app/env.py diff --git a/mftp/.dockerignore b/mftp/.dockerignore index f80b553..d8db2e5 100644 --- a/mftp/.dockerignore +++ b/mftp/.dockerignore @@ -16,6 +16,7 @@ __pycache__/ service/ .session .lsnif +.ntfy.lsnsf logs.txt *.json -env.* \ No newline at end of file +env.* diff --git a/mftp/README.md b/mftp/README.md index 71e9ec7..a44fedd 100644 --- a/mftp/README.md +++ b/mftp/README.md @@ -101,6 +101,7 @@ It is mandatory to provide either of the following flags to the execution comman -v /path/to/mftp_config/mail_send_token.json:/mftp/mail_send_token.json \ -v /path/to/mftp_config/mail_send_creds.json:/mftp/mail_send_creds.json \ -v /path/to/mftp_config/.lsnif:/mftp/.lsnif \ + -v /path/to/mftp_config/.ntfy.lsnsf:/mftp/.ntfy.lsnsf \ -v /path/to/mftp_config/.session:/mftp/.session \ --restart=unless-stopped \ --name mftp \ @@ -116,6 +117,7 @@ It is mandatory to provide either of the following flags to the execution comman -v /path/to/mftp_config/mail_send_token.json:/mftp/mail_send_token.json \ -v /path/to/mftp_config/mail_send_creds.json:/mftp/mail_send_creds.json \ -v /path/to/mftp_config/.lsnif:/mftp/.lsnif \ + -v /path/to/mftp_config/.ntfy.lsnsf:/mftp/.ntfy.lsnsf \ -v /path/to/mftp_config/.session:/mftp/.session \ --restart=unless-stopped \ --name mftp \ @@ -131,6 +133,7 @@ It is mandatory to provide either of the following flags to the execution comman -v /path/to/mftp_config/mail_send_token.json:/mftp/mail_send_token.json \ -v /path/to/mftp_config/mail_send_creds.json:/mftp/mail_send_creds.json \ -v /path/to/mftp_config/.lsnif:/mftp/.lsnif \ + -v /path/to/mftp_config/.ntfy.lsnsf:/mftp/.ntfy.lsnsf \ -v /path/to/mftp_config/.session:/mftp/.session \ --restart=unless-stopped \ --name mftp \ @@ -167,6 +170,7 @@ It is also possible to run these docker containers as a cronjob: -v /path/to/mftp_config/mail_send_token.json:/mftp/mail_send_token.json \ -v /path/to/mftp_config/mail_send_creds.json:/mftp/mail_send_creds.json \ -v /path/to/mftp_config/.lsnif:/mftp/.lsnif \ + -v /path/to/mftp_config/.ntfy.lsnsf:/mftp/.ntfy.lsnsf \ -v /path/to/mftp_config/.session:/mftp/.session \ --name mftp \ metakgporg/mftp --gmail-api --cron diff --git a/mftp/docker-compose.yml b/mftp/docker-compose.yml index 9136806..a86e8c0 100644 --- a/mftp/docker-compose.yml +++ b/mftp/docker-compose.yml @@ -1,21 +1,20 @@ - version: "3" services: - - mftp: - build: - context: . - dockerfile: Dockerfile - image: metakgporg/mftp - container_name: mftp - restart: unless-stopped - command: $MFTP_MODE - volumes: - - $MFTP_CONFIG/env.py:/mftp/env.py - - $MFTP_CONFIG/token.json:/mftp/token.json - - $MFTP_CONFIG/credentials.json:/mftp/credentials.json - - $MFTP_CONFIG/mail_send_token.json:/mftp/mail_send_token.json - - $MFTP_CONFIG/mail_send_creds.json:/mftp/mail_send_creds.json - - $MFTP_CONFIG/.lsnif:/mftp/.lsnif - - $MFTP_CONFIG/.session:/mftp/.session + mftp: + build: + context: . + dockerfile: Dockerfile + image: metakgporg/mftp + container_name: mftp + restart: unless-stopped + command: $MFTP_MODE + volumes: + - $MFTP_CONFIG/env.py:/mftp/env.py + - $MFTP_CONFIG/token.json:/mftp/token.json + - $MFTP_CONFIG/credentials.json:/mftp/credentials.json + - $MFTP_CONFIG/mail_send_token.json:/mftp/mail_send_token.json + - $MFTP_CONFIG/mail_send_creds.json:/mftp/mail_send_creds.json + - $MFTP_CONFIG/.lsnif:/mftp/.lsnif + - $MFTP_CONFIG/.ntfy.lsnsf:/mftp/.ntfy.lsnsf + - $MFTP_CONFIG/.session:/mftp/.session diff --git a/mftp/env.example.py b/mftp/env.example.py index 656f3e8..5357000 100644 --- a/mftp/env.example.py +++ b/mftp/env.example.py @@ -2,9 +2,9 @@ ROLL_NUMBER = "XXYYXXXXX" # Institute Roll Number PASSWORD = "**********" # ERP Password SECURITY_QUESTIONS_ANSWERS = { # ERP Secret Questions and their Answers - "Q1" : "A1", - "Q2" : "A2", - "Q3" : "A3", + "Q1": "A1", + "Q2": "A2", + "Q3": "A3", } # EMAIL (via SMTP) @@ -17,7 +17,20 @@ # NTFY NTFY_BASE_URL = "https://ntfy.sh" -NTFY_TOPIC = "mftp" +## This is a list of ntfy topics, with their respective filters, +## for the logic to determine which message is to be sent on which topic +NTFY_TOPICS = { + "mftp-test": {}, + "mftp-placement-test": { + "Type": "PLACEMENT", + }, + "mftp-internship-test": { + "Type": "INTERNSHIP", + }, + "mftp-ppo-test": { + "Subject": "PPO", + }, +} ## Optional: only if you want a custom icon NTFY_TOPIC_ICON = "https://miro.medium.com/v2/resize:fit:600/1*O94LHxqfD_JGogOKyuBFgA.jpeg" ## Optional: only if the topic is restricted diff --git a/mftp/notice.py b/mftp/notice.py index bd9fb57..fe82a54 100644 --- a/mftp/notice.py +++ b/mftp/notice.py @@ -47,6 +47,59 @@ def fetch(headers, session, ssoToken, lsnif): return notices +def get_latest_subscribers(lsnsf): + try: + with open(lsnsf, 'r') as file: + successful_subscribers= file.read() + + return successful_subscribers.split(' ') + except Exception as e: + logging.error(f" Failed to Read `{lsnsf}` file") + + +def reset_lsns(lsnsf): + try: + with open(lsnsf, 'w') as file: + file.write(f'') + except Exception as e: + logging.error(f" Failed to Reset `{lsnsf}` file") + + +def update_lsns(lsnsf, ntfy_topic): + # Create file if it doesn't exist + if not os.path.exists(lsnsf): + open(lsnsf, 'w').close() + + # Save the value of Latest Sent Notice Subscriber in the list + # which is a list os subscribers separated by space + try: + with open(lsnsf, 'r') as file: + existing_subscribers= file.read() + + with open(lsnsf, 'w') as file: + file.write(f'{ntfy_topic} {existing_subscribers}') + except Exception as e: + logging.error(f" Failed to Save Subscriber ~ {ntfy_topic}") + + +def filter_subscribers(notice, subscribers): + filtered_subscribers = [] + + for subscriber in subscribers: + filters = subscribers[subscriber] + + if len(filters) == 0: + filtered_subscribers.append(subscriber) + + for filter in filters: + filter_value = filters[filter] + + if notice[filter] == filter_value: + filtered_subscribers.append(subscriber) + + return filtered_subscribers + + def get_latest_index(lsnif): try: with open(lsnif, 'r') as file: diff --git a/mftp/ntfy.py b/mftp/ntfy.py index a63c667..1e58a28 100644 --- a/mftp/ntfy.py +++ b/mftp/ntfy.py @@ -5,9 +5,11 @@ import requests from urllib.parse import quote from bs4 import BeautifulSoup as bs -from notice import update_lsni, has_idx_mutated from endpoints import NOTICE_CONTENT_URL, ATTACHMENT_URL -from env import NTFY_BASE_URL, NTFY_TOPIC, NTFY_TOPIC_ICON, NTFY_USER, NTFY_PASS, HEIMDALL_COOKIE +from env import NTFY_BASE_URL, NTFY_TOPICS, NTFY_TOPIC_ICON, NTFY_USER, NTFY_PASS, HEIMDALL_COOKIE +from notice import filter_subscribers, get_latest_subscribers, reset_lsns, update_lsni, has_idx_mutated, update_lsns + +lsnsf = '.ntfy.lsnsf' def ntfy_priority(subject): match subject: @@ -76,6 +78,10 @@ def format_notice(notices, session): "Attachment": f"{id_}-{notice['Type']}-{notice['Subject']}-{notice['Company']}.pdf".replace(' ', '_').replace('/', '_') } + # NTFY TOPICS LIST: Based on filters + notification["NTFY_TOPICS"] = filter_subscribers(notice, NTFY_TOPICS) + + # Handling attachments try: attachment = parseAttachment(session, year, id_) except Exception as e: @@ -96,52 +102,66 @@ def format_notice(notices, session): def send(notifications, lsnif, notices): if notifications: - print(f"[SENDING NOTIFICATIONS] ~ {NTFY_BASE_URL}/{NTFY_TOPIC}", flush=True) + print(f"[SENDING NOTIFICATIONS]", flush=True) for i, notification in enumerate(notifications, 1): if has_idx_mutated(lsnif, notices, i): break - try: - query_params = f"message={quote(notification['Body'])}" - request_url = f"{NTFY_BASE_URL}/{NTFY_TOPIC}?{query_params}" - - headers={ - "Title": notification["Title"], - "Tags": notification["Tags"], - "Priority": notification["Priority"], - "Icon": NTFY_TOPIC_ICON, - "Action": notification["Links"], - "Markdown": "false" - } - if NTFY_USER and NTFY_PASS: - headers['Authorization'] = f"Basic {str(base64.b64encode(bytes(NTFY_USER + ':' + NTFY_PASS, 'utf-8')), 'utf-8')}" - - cookies = {} - if HEIMDALL_COOKIE: - cookies = {'heimdall': HEIMDALL_COOKIE} - - if notification['Attachment']: - headers['Filename'] = notification['Attachment'] - response = requests.put( - request_url, - headers=headers, - data=open(notification['Attachment'], 'rb'), - cookies=cookies - ) - else: - response = requests.put(request_url, headers=headers, cookies=cookies) - - except Exception as e: - logging.error(f" Failed to request NTFY SERVER: {notification['Title']} ~ {str(e)}") - break - finally: - if notification['Attachment'] and not delete_file(notification['Attachment']): break - - if response.status_code == 200: - logging.info(f" [NOTIFICATION SENT] ~ {notification['Title']}") + notification_sent_to_all_subscribers = True + + ntfy_topics = notification['NTFY_TOPICS'] + latest_successful_subscribers = get_latest_subscribers(lsnsf) + if len(latest_successful_subscribers) != 0: + ntfy_topics = [subscirber for subscirber in ntfy_topics if subscirber not in latest_successful_subscribers] + + for ntfy_topic in ntfy_topics: + try: + query_params = f"message={quote(notification['Body'])}" + request_url = f"{NTFY_BASE_URL}/{ntfy_topic}?{query_params}" + + headers={ + "Title": notification["Title"], + "Tags": notification["Tags"], + "Priority": notification["Priority"], + "Icon": NTFY_TOPIC_ICON, + "Action": notification["Links"], + "Markdown": "false" + } + if NTFY_USER and NTFY_PASS: + headers['Authorization'] = f"Basic {str(base64.b64encode(bytes(NTFY_USER + ':' + NTFY_PASS, 'utf-8')), 'utf-8')}" + + cookies = {} + if HEIMDALL_COOKIE: + cookies = {'heimdall': HEIMDALL_COOKIE} + + if notification['Attachment']: + headers['Filename'] = notification['Attachment'] + response = requests.put( + request_url, + headers=headers, + data=open(notification['Attachment'], 'rb'), + cookies=cookies + ) + else: + response = requests.put(request_url, headers=headers, cookies=cookies) + except Exception as e: + logging.error(f" Failed to request NTFY SERVER: {notification['Title']} ~ {str(e)}") + break + finally: + if notification['Attachment'] and not delete_file(notification['Attachment']): break + + if response.status_code == 200: + logging.info(f" [NOTIFICATION SENT] ~ `{notification['Title'].split(' | ')[0]} -> {ntfy_topic}`") + update_lsns(lsnsf, ntfy_topic) + else: + logging.error(f" Failed to send notification: `{notification['Title'].split(' | ')[0]} -> {ntfy_topic}` ~ {response.text}") + notification_sent_to_all_subscribers = False + break + + if notification_sent_to_all_subscribers: + reset_lsns(lsnsf) update_lsni(lsnif, notices, i) else: - logging.error(f" Failed to send notification: {notification['Title']} ~ {response.text}") break def save_file(file_name: str, attachment):