diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0cf99dc --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[report] +sort = Cover +[run] +omit = src/slacknewsbot/venv/* \ No newline at end of file diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..12558c8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +[flake8] +max-line-length = 88 +ignore = E501,E402,E127 +exclude = + venv + .terraform diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..5d7b861 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,32 @@ +name: CI + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f src/slacknewsbot/requirements.txt ]; then pip install -r src/slacknewsbot/requirements.txt; fi + + - name: Lint with flake8 + run: | + flake8 --count --show-source --statistics + + - name: Test with pytest + run: | + PYTHONPATH=./src pytest --cov=src --cov-branch --cov-report term-missing \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..459e06f --- /dev/null +++ b/.gitignore @@ -0,0 +1,116 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +builds/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cachepip install -r src/slacknewsbot/requirements.txt +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +.vscode/ + +*.git +*.tfstate +*.backup +*.tfplan +.terraform* +*.zip +*output.txt +secrets* \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..abac715 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: git://github.com/antonbabenko/pre-commit-terraform + rev: v1.45.0 + hooks: + - id: terraform_fmt diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..184d2d0 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +.PHONY: check test deploy deploy-with-secrets-file + +check: ## Run linters + @echo "*** running linters ***" + flake8 --count --show-source --statistics + @echo "*** all linters passing ***" +test: check ## Run tests + @echo "*** running tests ***" + PYTHONPATH=./src pytest --cov=src --cov-branch --cov-report term-missing + @echo "*** all tests passing ***" +deploy: test ## Deploy project when secrets are exported as env variables + @echo "*** running deploy ***" + terraform apply +deploy-with-secrets-file: test ## Deploy project when secrets are stored in secrets.tfvars file + @echo "*** running deploy ***" + terraform apply -var-file=secrets.tfvars \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f61721c --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ + + +## Create virtualenv +``` +python3 -m venv venv +source venv/bin/activate +pip install -r src/slacknewsbot/requirements.txt +``` + +## Run tests +``` +make tests +``` + +## Deploy function +### Export secrets +``` +export TF_VAR_slack_bot_token=XXX +export TF_VAR_ph_api_token=XXX +``` +### Deploy +``` +make deploy +``` +### Deploy with secrets file +``` +make deploy-with-secrets-file +``` \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..da33df4 --- /dev/null +++ b/main.tf @@ -0,0 +1,50 @@ +provider "aws" { + region = "eu-west-1" +} + + +module "lambda_function" { + source = "terraform-aws-modules/lambda/aws" + version = "2.2.0" + + function_name = var.name + description = "Function to post popular articles to Slack" + handler = "app.lambda_handler" + runtime = "python3.8" + timeout = 30 + + source_path = "./src/slacknewsbot" + + environment_variables = { + SLACK_CHANNEL = var.slack_channel_name + SLACK_BOT_TOKEN = var.slack_bot_token + QUERY_HN = var.query_hn + QUERY_PH = var.query_ph + PH_API_TOKEN = var.ph_api_token + STORIES_NUMBER = var.stories_number + } + allowed_triggers = { + PostToSlack = { + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.cw_cron.arn + } + } + create_current_version_allowed_triggers = false + + tags = { + Name = var.name + } +} + +# EVENT CONFIG +resource "aws_cloudwatch_event_rule" "cw_cron" { + name = "${name}-lambda-trigger" + description = "Lambda trigge to post to Slack" + + schedule_expression = "cron(0 13 * * ? *)" +} + +resource "aws_cloudwatch_event_target" "trigger_slack" { + rule = aws_cloudwatch_event_rule.cw_cron.name + arn = module.lambda_function.lambda_function_arn +} diff --git a/src/slacknewsbot/__init__.py b/src/slacknewsbot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/slacknewsbot/app.py b/src/slacknewsbot/app.py new file mode 100644 index 0000000..a2bf95b --- /dev/null +++ b/src/slacknewsbot/app.py @@ -0,0 +1,166 @@ +import os +import logging +import asyncio +from aiohttp import ClientSession +import requests +import json +from functools import wraps + +from slack_sdk import WebClient + + +HN_API_URL = os.environ.get("HN_API_URL", "https://hacker-news.firebaseio.com/v0") +HN_URL = os.environ.get("HN_URL", "https://news.ycombinator.com") +PH_API_URL = os.environ.get("PH_API_URL", "https://api.producthunt.com/v2/api/graphql") +PH_API_TOKEN = os.environ.get("PH_API_TOKEN", "") +STORIES_NUMBER = int(os.environ.get("STORIES_NUMBER", 3)) + + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +logger = logging.getLogger("Logger") +logger.setLevel(os.environ.get("LOGGING", logging.DEBUG)) + + +def notify_cloudwatch(function): + @wraps(function) + def wrapper(event, context): + logger.info(f"'{context.function_name}' - entry:'{event}'") + result = function(event, context) + logger.info(f"'{context.function_name}' - entry.'{result}'") + return result + return wrapper + + +class GetHN: + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + self.logger.setLevel(os.environ.get("LOGGING", logging.DEBUG)) + + async def fetch(self, session, url): + """Fetch details for a single story""" + async with session.get(url) as response: + if response.status != 200: + response.raise_for_status() + response = await response.text() + return json.loads(response) + + async def fetch_all(self, urls): + """Fetch details for all top stories""" + async with ClientSession() as session: + tasks = [] + for url in urls: + task = asyncio.create_task(self.fetch(session, url)) + tasks.append(task) + results = await asyncio.gather(*tasks) + return results + + def get_top_stories(self): + """Get a list of top stories with details""" + stories = requests.get(f"{HN_API_URL}/topstories.json") + stories_ids = json.loads(stories.text) + urls = [f"{HN_API_URL}/item/{story_id}.json" for story_id in stories_ids] + fetched_stories = asyncio.run(self.fetch_all(urls)) + sorted_stories = sorted( + fetched_stories, key=lambda k: k["score"], reverse=True) + return sorted_stories[:STORIES_NUMBER] + + def create_hn_text(self): + """Create slack text with HackerNews top stories""" + text_list = [f"Top {STORIES_NUMBER} from HackerNews:"] + sorted_stories = self.get_top_stories() + # Format slack text + for story in sorted_stories: + text_list.append( + "*<{}|{}>* - <{}|{}>".format( + "{}/item?id={}".format(HN_URL, story["id"]), + story["score"], + story["url"], + story["title"], + ) + ) + self.logger.debug(text_list) + return "\n>".join(text_list) + + +class GetPH: + + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + self.logger.setLevel(os.environ.get("LOGGING", logging.DEBUG)) + + def run_graphql_query( + self, + query, + headers, + status_code=200): + request = requests.post(PH_API_URL, data=json.dumps(query), headers=headers) + if request.status_code == status_code: + return request.json() + else: + raise Exception( + "Unexpected status code returned: {}".format( + request.status_code) + ) + + def create_ph_text(self): + text_list = [f"Top {STORIES_NUMBER} from Product Hunt:"] + query = { + "query": """ + query todayPosts { + posts { + edges { + node { + name + tagline + votesCount + website + url + } + } + } + } + """ + } + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + "Authorization": "Bearer " + PH_API_TOKEN, + } + response = self.run_graphql_query(query, headers) + today_posts = [ + post["node"] for post in response["data"]["posts"]["edges"]] + top_posts = sorted( + today_posts, key=lambda k: k["votesCount"], reverse=True) + # Format slack text + for post in top_posts[:STORIES_NUMBER]: + text_list.append( + "*<{}|{}>* - <{}|{} - {}>".format( + post["url"], + post["votesCount"], + post["website"], + post["name"], + post["tagline"], + ) + ) + self.logger.debug(text_list) + return "\n>".join(text_list) + + +def post_msg(text): + client = WebClient(token=os.environ["SLACK_BOT_TOKEN"]) + client.chat_postMessage( + channel=os.environ["SLACK_CHANNEL"], + text="News", + blocks=[ + {"type": "section", "text": {"type": "mrkdwn", "text": (text)}}], + ) + return text + + +@notify_cloudwatch +def lambda_handler(event, context): + if os.environ["QUERY_HN"]: + post_msg(GetHN().create_hn_text()) + if os.environ["QUERY_PH"]: + post_msg(GetPH().create_ph_text()) + return "SUCCESS" diff --git a/src/slacknewsbot/requirements.txt b/src/slacknewsbot/requirements.txt new file mode 100644 index 0000000..7fab9a9 --- /dev/null +++ b/src/slacknewsbot/requirements.txt @@ -0,0 +1,21 @@ +aiohttp==3.7.4.post0 +async-timeout==3.0.1 +attrs==21.2.0 +certifi==2020.12.5 +chardet==4.0.0 +coverage==5.5 +idna==2.10 +iniconfig==1.1.1 +multidict==5.1.0 +packaging==20.9 +pluggy==0.13.1 +py==1.10.0 +pyparsing==2.4.7 +pytest==6.2.4 +pytest-cov==2.12.0 +requests==2.25.1 +slack-sdk==3.5.1 +toml==0.10.2 +typing-extensions==3.10.0.0 +urllib3==1.26.4 +yarl==1.6.3 diff --git a/src/tests/unit/test_get_hn.py b/src/tests/unit/test_get_hn.py new file mode 100644 index 0000000..4900b39 --- /dev/null +++ b/src/tests/unit/test_get_hn.py @@ -0,0 +1,29 @@ +from unittest.mock import MagicMock + +from pytest import fixture + +from slacknewsbot.app import GetHN + + +SLACK_TEXT = "Top 3 from HackerNews:\n>** - " + + +@fixture() +def obj(): + obj = GetHN.__new__(GetHN) + obj.logger = MagicMock() + return obj + + +def test_should_create_hn_text(obj): + obj.get_top_stories = MagicMock( + return_value=[{ + "id": 1, + "score": 1, + "url": "abc", + "title": "bcd" + }] + ) + assert obj.create_hn_text() == SLACK_TEXT + +# TODO: Add more tests diff --git a/state.tf b/state.tf new file mode 100644 index 0000000..22de3e5 --- /dev/null +++ b/state.tf @@ -0,0 +1,9 @@ +terraform { + + backend "s3" { + bucket = "mano-tf-projects" + key = "slack-news-bot/main.tfstate" + region = "eu-west-1" + encrypt = true + } +} \ No newline at end of file diff --git a/terraform.tfvars b/terraform.tfvars new file mode 100644 index 0000000..1469d6b --- /dev/null +++ b/terraform.tfvars @@ -0,0 +1,4 @@ +name = "slack-news-bot" +slack_channel_name = "news" +query_hn = "True" +query_ph = "True" \ No newline at end of file diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..e47a315 --- /dev/null +++ b/variables.tf @@ -0,0 +1,31 @@ +variable "name" { + description = "Project name" + type = string +} +variable "slack_channel_name" { + description = "The name of the Slack channel where messages are posted" + type = string +} +variable "slack_bot_token" { + description = "Slack bot token to post to channel" + type = string + sensitive = true +} +variable "query_hn" { + description = "Controls whether to get top stories from Hacker News" + type = string +} +variable "query_ph" { + description = "Controls whether to get top stories from Product Hunt. If set to True, need to specify value for ph_api_token variable" + type = string +} +variable "ph_api_token" { + description = "Product Hunt API token. Need to specify value by exporting environment variable or using secrets.tfvars" + type = string + sensitive = true + default = "" +} +variable "stories_number" { + description = "Number of stories to post to Slack" + type = string +} \ No newline at end of file