From 2bafb1b99b552e2817484adea21cab087077dda0 Mon Sep 17 00:00:00 2001 From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com> Date: Wed, 28 Dec 2022 10:00:43 -0500 Subject: [PATCH 1/2] Consolidate flake8 settings Consolidates sane flake8 settings to pyproject with the Flake8-pyproject plugin. Does not change logic of test workflow but allows base settings to live in pyproject for anyone using flake8 as an external linter (e.g. vscode) Also fixes some flake8 errors that were not being picked up by current testing, mostly unused imports. --- .github/workflows/test.yml | 8 ++++++-- README.md | 8 ++++++-- bdfr/connector.py | 4 ++-- bdfr/site_downloaders/imgur.py | 2 +- bdfr/site_downloaders/vreddit.py | 7 ++----- pyproject.toml | 6 ++++++ tests/test_download_filter.py | 2 +- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d52ef63..ca32bb3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,8 +3,12 @@ name: Python Test on: push: branches: [ master, development ] + paths-ignore: + - "*.md" pull_request: branches: [ master, development ] + paths-ignore: + - "*.md" jobs: test: @@ -27,7 +31,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip flake8 pytest pytest-cov + python -m pip install --upgrade pip Flake8-pyproject pytest pytest-cov pip install . - name: Make configuration for tests @@ -38,7 +42,7 @@ jobs: - name: Lint with flake8 run: | - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --select=E9,F63,F7,F82 - name: Test with pytest run: | diff --git a/README.md b/README.md index 120448e3..dd700ad1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # Bulk Downloader for Reddit -[![PyPI version](https://img.shields.io/pypi/v/bdfr.svg)](https://pypi.python.org/pypi/bdfr) -[![PyPI downloads](https://img.shields.io/pypi/dm/bdfr)](https://pypi.python.org/pypi/bdfr) +[![PyPI Status](https://img.shields.io/pypi/status/bdfr?logo=PyPI)](https://pypi.python.org/pypi/bdfr) +[![PyPI version](https://img.shields.io/pypi/v/bdfr.svg?logo=PyPI)](https://pypi.python.org/pypi/bdfr) +[![PyPI downloads](https://img.shields.io/pypi/dm/bdfr?logo=PyPI)](https://pypi.python.org/pypi/bdfr) +[![AUR version](https://img.shields.io/aur/version/python-bdfr?logo=Arch%20Linux)](https://aur.archlinux.org/packages/python-bdfr) [![Python Test](https://github.com/aliparlakci/bulk-downloader-for-reddit/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/aliparlakci/bulk-downloader-for-reddit/actions/workflows/test.yml) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?logo=Python)](https://github.com/psf/black) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) This is a tool to download submissions or submission data from Reddit. It can be used to archive data or even crawl Reddit to gather research data. The BDFR is flexible and can be used in scripts if needed through an extensive command-line interface. [List of currently supported sources](#list-of-currently-supported-sources) diff --git a/bdfr/connector.py b/bdfr/connector.py index bf50f32d..d0c4ac23 100644 --- a/bdfr/connector.py +++ b/bdfr/connector.py @@ -205,7 +205,7 @@ def create_file_logger(self): else: log_path = Path(self.args.log).resolve().expanduser() if not log_path.parent.exists(): - raise errors.BulkDownloaderException(f"Designated location for logfile does not exist") + raise errors.BulkDownloaderException("Designated location for logfile does not exist") backup_count = self.cfg_parser.getint("DEFAULT", "backup_log_count", fallback=3) file_handler = logging.handlers.RotatingFileHandler( log_path, @@ -323,7 +323,7 @@ def determine_sort_function(self) -> Callable: def get_multireddits(self) -> list[Iterator]: if self.args.multireddit: if len(self.args.user) != 1: - logger.error(f"Only 1 user can be supplied when retrieving from multireddits") + logger.error("Only 1 user can be supplied when retrieving from multireddits") return [] out = [] for multi in self.split_args_input(self.args.multireddit): diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py index a4c378f3..537ab644 100644 --- a/bdfr/site_downloaders/imgur.py +++ b/bdfr/site_downloaders/imgur.py @@ -67,7 +67,7 @@ def _get_data(link: str) -> dict: image_dict = re.search(outer_regex, chosen_script).group(1) image_dict = re.search(inner_regex, image_dict).group(1) except AttributeError: - raise SiteDownloaderError(f"Could not find image dictionary in page source") + raise SiteDownloaderError("Could not find image dictionary in page source") try: image_dict = json.loads(image_dict) diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py index 8f6022e9..48f5ba1f 100644 --- a/bdfr/site_downloaders/vreddit.py +++ b/bdfr/site_downloaders/vreddit.py @@ -2,14 +2,11 @@ # -*- coding: utf-8 -*- import logging -import tempfile -from pathlib import Path -from typing import Callable, Optional +from typing import Optional -import yt_dlp from praw.models import Submission -from bdfr.exceptions import NotADownloadableLinkError, SiteDownloaderError +from bdfr.exceptions import NotADownloadableLinkError from bdfr.resource import Resource from bdfr.site_authenticator import SiteAuthenticator from bdfr.site_downloaders.youtube import Youtube diff --git a/pyproject.toml b/pyproject.toml index 6879a6d1..a8b0dd21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,12 @@ bdfr-download = "bdfr.__main__:cli_download" [tool.black] line-length = 120 +[tool.flake8] +exclude = ["scripts"] +max-line-length = 120 +show-source = true +statistics = true + [tool.isort] profile = "black" py_version = 39 diff --git a/tests/test_download_filter.py b/tests/test_download_filter.py index 3b4d6b80..6062dc3e 100644 --- a/tests/test_download_filter.py +++ b/tests/test_download_filter.py @@ -76,4 +76,4 @@ def test_filter_empty_filter(test_url: str): download_filter = DownloadFilter() test_resource = Resource(MagicMock(), test_url, lambda: None) result = download_filter.check_resource(test_resource) - assert result is True + assert result From c4bece2f5844e41e4771c7a95b857474ac0ab030 Mon Sep 17 00:00:00 2001 From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com> Date: Sat, 31 Dec 2022 09:09:48 -0500 Subject: [PATCH 2/2] Add flake8 to precommit adds flake8 to pre-commit and dev requirements. --- .pre-commit-config.yaml | 6 ++++++ docs/CONTRIBUTING.md | 3 ++- pyproject.toml | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7013228d..add4ea63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,12 @@ repos: - id: isort name: isort (python) + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: [Flake8-pyproject] + - repo: https://github.com/markdownlint/markdownlint rev: v0.12.0 hooks: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 72666e77..53fcb03a 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -69,12 +69,13 @@ python3 -m pip install -e .[dev] The BDFR project uses several tools to manage the code of the project. These include: - [black](https://github.com/psf/black) +- [flake8](https://github.com/john-hen/Flake8-pyproject) - [isort](https://github.com/PyCQA/isort) - [markdownlint (mdl)](https://github.com/markdownlint/markdownlint) - [tox](https://tox.wiki/en/latest/) - [pre-commit](https://github.com/pre-commit/pre-commit) -The first three tools are formatters. These change the code to the standards expected for the BDFR project. The configuration details for these tools are contained in the [pyproject.toml](../pyproject.toml) file for the project. +The first four tools are formatters. These change the code to the standards expected for the BDFR project. The configuration details for these tools are contained in the [pyproject.toml](../pyproject.toml) file for the project. The tool `tox` is used to run tests and tools on demand and has the following environments: diff --git a/pyproject.toml b/pyproject.toml index a8b0dd21..4a28a8f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ data-files = {"config" = ["bdfr/default_config.cfg",]} [project.optional-dependencies] dev = [ "black>=22.10.0", + "Flake8-pyproject>=1.2.2", "isort>=5.10.1", "pre-commit>=2.20.0", "pytest>=7.1.0",