-
-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f310f2a
commit 3702ae5
Showing
18 changed files
with
832 additions
and
344 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| TARGET_URL=https://hackersandslackers.com |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| [flake8] | ||
| select = E9,F63,F7,F82 | ||
| exclude = .git,.github,__pycache__,.pytest_cache,.venv,logs,creds,.reports | ||
| max-line-length = 120 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -106,8 +106,10 @@ venv.bak/ | |
| credentials.json | ||
| gcloud.json | ||
|
|
||
| # Etc. | ||
| .idea | ||
| .pytest_cache | ||
| # OS | ||
| .DS_Store | ||
|
|
||
| # IDEs | ||
| .idea | ||
| .vs_code | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| PROJECT_NAME := $(shell basename $CURDIR) | ||
| VIRTUAL_ENV := $(CURDIR)/.venv | ||
| LOCAL_PYTHON := $(VIRTUAL_ENV)/bin/python3 | ||
|
|
||
| define HELP | ||
| Manage $(PROJECT_NAME). Usage: | ||
|
|
||
| make run - Run $(PROJECT_NAME) locally. | ||
| make install - Create local virtualenv & install dependencies. | ||
| make deploy - Set up project & run locally. | ||
| make update - Update dependencies via Poetry and output resulting `requirements.txt`. | ||
| make format - Run Python code formatter & sort dependencies. | ||
| make lint - Check code formatting with flake8. | ||
| make clean - Remove extraneous compiled files, caches, logs, etc. | ||
|
|
||
| endef | ||
| export HELP | ||
|
|
||
|
|
||
| .PHONY: run install deploy update format lint clean help | ||
|
|
||
| all help: | ||
| @echo "$$HELP" | ||
|
|
||
| env: $(VIRTUAL_ENV) | ||
|
|
||
| $(VIRTUAL_ENV): | ||
| if [ ! -d $(VIRTUAL_ENV) ]; then \ | ||
| echo "Creating Python virtual env in \`${VIRTUAL_ENV}\`"; \ | ||
| python3 -m venv $(VIRTUAL_ENV); \ | ||
| fi | ||
| poetry config virtualenvs.path $(VIRTUAL_ENV) | ||
|
|
||
| .PHONY: run | ||
| run: env | ||
| $(LOCAL_PYTHON) -m main | ||
|
|
||
| .PHONY: install | ||
| install: env | ||
| $(shell . $(VIRTUAL_ENV)/bin/activate) | ||
| $(LOCAL_PYTHON) -m pip install --upgrade pip setuptools wheel && \ | ||
| poetry install --with dev --sync | ||
| echo Installed dependencies in \`${VIRTUAL_ENV}\`; | ||
|
|
||
| .PHONY: deploy | ||
| deploy: | ||
| make install && \ | ||
| make run | ||
|
|
||
| .PHONY: update | ||
| update: env | ||
| $(LOCAL_PYTHON) -m pip install --upgrade pip setuptools wheel && \ | ||
| poetry update --with dev && \ | ||
| poetry export -f requirements.txt --output requirements.txt --without-hashes && \ | ||
| echo Installed dependencies in \`${VIRTUAL_ENV}\`; | ||
|
|
||
| .PHONY: format | ||
| format: env | ||
| $(LOCAL_PYTHON) -m isort --multi-line=3 . && \ | ||
| $(LOCAL_PYTHON) -m black . | ||
|
|
||
| .PHONY: lint | ||
| lint: env | ||
| $(LOCAL_PYTHON) -m flake8 . --count \ | ||
| --select=E9,F63,F7,F82 \ | ||
| --exclude .git,.github,__pycache__,.pytest_cache,.venv,logs,creds,.venv,docs,logs,.reports \ | ||
| --show-source \ | ||
| --statistics | ||
|
|
||
| .PHONY: clean | ||
| clean: | ||
| find . -name 'poetry.lock' -delete && \ | ||
| find . -name '.coverage' -delete && \ | ||
| find . -name '.Pipfile.lock' -delete && \ | ||
| find . -wholename '**/*.pyc' -delete && \ | ||
| find . -type d -wholename '__pycache__' -exec rm -rf {} + && \ | ||
| find . -type d -wholename './.venv' -exec rm -rf {} + && \ | ||
| find . -type d -wholename '.pytest_cache' -exec rm -rf {} + && \ | ||
| find . -type d -wholename '**/.pytest_cache' -exec rm -rf {} + && \ | ||
| find . -type d -wholename './logs/*.log' -exec rm -rf {} + && \ | ||
| find . -type d -wholename './.reports/*' -exec rm -rf {} + |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| """Scrape metadata from target URL.""" | ||
| import pprint | ||
|
|
||
| from beautifulsoup_tutorial.fetch import fetch_html_from_url | ||
| from beautifulsoup_tutorial.scrape import scrape_page_metadata | ||
|
|
||
| from config import TARGET_URL | ||
|
|
||
|
|
||
| def init_script() -> dict: | ||
| """ | ||
| Fetch a given HTML page to extract & display metadata for. | ||
| returns: dict | ||
| """ | ||
| resp = fetch_html_from_url(TARGET_URL) | ||
| metadata = scrape_page_metadata(resp, TARGET_URL) | ||
| pp = pprint.PrettyPrinter(indent=4, width=120, sort_dicts=False) | ||
| pp.pprint(metadata) | ||
| return metadata |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| """Fetch raw HTML from a URL.""" | ||
| from typing import Optional | ||
|
|
||
| import requests | ||
| from requests.exceptions import HTTPError | ||
|
|
||
|
|
||
| def fetch_html_from_url(url: str) -> Optional[str]: | ||
| """ | ||
| Fetch raw HTML from a URL. | ||
| :param str url: URL to `GET` contents from. | ||
| :return: Optional[str] | ||
| """ | ||
| try: | ||
| headers = { | ||
| "Access-Control-Allow-Origin": "*", | ||
| "Access-Control-Allow-Methods": "GET", | ||
| "Access-Control-Allow-Headers": "Content-Type", | ||
| "Access-Control-Max-Age": "3600", | ||
| "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0", | ||
| } | ||
| return requests.get(url, headers=headers) | ||
| except HTTPError as e: | ||
| print(f"HTTP error occurred: {e}") | ||
| except Exception as e: | ||
| print(f"Unexpected error occurred: {e}") |
Oops, something went wrong.