Skip to content

Commit

Permalink
Modernize Python tooling
Browse files Browse the repository at this point in the history
Move all configs into `pyproject.toml`, switch test/packaging to
use Hatch, and do all the linting with Ruff.

Fix issues that Ruff turned up, and annotate/ignore the pieces that
it got wrong.

Fixes: Issue #33
  • Loading branch information
nisimond committed Feb 28, 2024
1 parent 7d9d430 commit 4f57896
Show file tree
Hide file tree
Showing 22 changed files with 139 additions and 136 deletions.
4 changes: 0 additions & 4 deletions .coveragerc

This file was deleted.

4 changes: 2 additions & 2 deletions .github/workflows/python-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
pip install hatch twine
- name: Build and publish
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python setup.py sdist bdist_wheel
hatch build
twine upload dist/*
6 changes: 3 additions & 3 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,15 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e .[tests]
pip install hatch
- name: Lint
run: |
pycodestyle --ignore=E252,W503,W504 spinach tests
hatch run pep8
- name: Test with pytest
run: |
pytest -v --cov=spinach tests/
hatch run py3
- name: Coveralls
uses: AndreMiras/coveralls-python-action@develop
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# _version is generated by setuptools_scm and requests not to be under a VCS
spinach/_version.py

.hatch
.pyc
__pycache__

Expand Down
5 changes: 3 additions & 2 deletions doc/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
sys.path.insert(0, os.path.abspath('..'))

import spinach
from spinach import _version as spinach_version

# -- General configuration ------------------------------------------------

Expand Down Expand Up @@ -66,9 +67,9 @@
# built documents.
#
# The short X.Y version.
version = spinach.__version__
version = spinach_version.__version__
# The full version, including alpha/beta/rc tags.
release = spinach.__version__
release = spinach_version.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
Expand Down
113 changes: 113 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
[build-system]
requires = ["hatchling", "hatch-vcs"]
build-backend = "hatchling.build"

[project]
name = "spinach"
dynamic = ["version"]
description = "Modern Redis task queue for Python 3"
readme = "README.rst"
license-files = { paths = ["LICENSE"] }
requires-python = ">=3.8.0,<4.0.0"
authors = [
{ name = "Nicolas Le Manchet", email = "nicolas@lemanchet.fr" },
]
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"Topic :: Software Development :: Libraries",
"Topic :: System :: Distributed Computing",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
]
keywords = ["task", "queue", "jobs", "redis"]
urls.Source = "https://github.com/NicolasLM/spinach"

dependencies = [
"redis",
"blinker"
]

[project.optional-dependencies]
tests = [
"pytest",
"pytest-cov",
"pytest-threadleak",
"ruff",
"flask",
"django"
]

[tool.hatch.version]
source = "vcs"

[tool.hatch.build.hooks.vcs]
version-file = "spinach/_version.py"

[tool.hatch.build.targets.sdist]
ignore-vcs = true
include = ["/spinach", "/tests"]

[tool.hatch.build.targets.wheel]
packages = ["spinach"]

[tool.hatch.build.targets.wheel.package-data]
"spinach.brokers.redis_scripts" = [
"deregister.lua",
"enqueue_job.lua",
"enqueue_jobs_from_dead_broker.lua",
"flush.lua",
"get_jobs_from_queue.lua",
"move_future_jobs.lua",
"register_periodic_tasks.lua",
"remove_job_from_running.lua",
"set_concurrency_keys.lua",
]

[tool.hatch.envs.default]
path = ".hatch"
features = ["tests"]

scripts.pep8 = ["ruff spinach tests"]

scripts.py3 = [
"docker-compose -f tests/docker-compose.yml up -d",
"pytest tests {args}",
"docker-compose -f tests/docker-compose.yml down",
]

scripts.ci = ["py3", "- pep8"]

[tool.ruff.lint]
select = ["B", "C9", "D", "E", "F", "S", "W"]
ignore = [
"B018", # Useless expression (seems to have false-positive bugs)
"S101", # Use of `assert` detected (conflicts with pytest)
"D10", # Missing docstring in public function / module / etc.
"D203", # 1 blank line required before class docstring
"D205", # 1 blank line required between summary line and description
"D213", # Multi-line docstring summary should start at the second line
"D40", # First line of docstring complaints
"D415", # First line should end with a period, question mark, or exclamation point
]
mccabe.max-complexity = 13

[tool.ruff.lint.per-file-ignores]
# Aggregating symbols into a convenient location for import is good practice
# for libraries, we don't care if they look unused
"__init__.py" = ["F401"]

[tool.coverage.run]
relative_files = true
include = ["spinach/*"]
omit = ["spinach/_version.py"]

[tool.pytest.ini_options]
threadleak = true
2 changes: 0 additions & 2 deletions pytest.ini

This file was deleted.

75 changes: 0 additions & 75 deletions setup.py

This file was deleted.

3 changes: 0 additions & 3 deletions spinach/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
from .brokers.memory import MemoryBroker
from .brokers.redis import RedisBroker
from .const import VERSION
from .engine import Engine
from .task import Tasks, Batch, RetryException, AbortException
from .worker import ThreadWorkers, AsyncioWorkers

__version__ = VERSION
2 changes: 1 addition & 1 deletion spinach/brokers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def wait_for_event(self):
if self._something_happened.wait(timeout=timeout):
self._something_happened.clear()

def start(self):
def start(self): # noqa: B027
"""Start the broker.
Only needed by arbiter.
Expand Down
2 changes: 0 additions & 2 deletions spinach/const.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
VERSION = '0.0.20'

DEFAULT_QUEUE = 'spinach'
DEFAULT_NAMESPACE = 'spinach'
DEFAULT_MAX_RETRIES = 0
Expand Down
4 changes: 2 additions & 2 deletions spinach/contrib/flask_spinach.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,14 @@ def spin(self):
return flask.current_app.extensions['spinach']
except (AttributeError, TypeError, KeyError):
raise RuntimeError('Spinach extension not initialized. '
'Did you forget to call init_app?')
'Did you forget to call init_app?') from None

def register_tasks(self, app, tasks):
try:
app.extensions['spinach'].attach_tasks(tasks)
except KeyError:
raise RuntimeError('Spinach extension not initialized. '
'Did you forget to call init_app?')
'Did you forget to call init_app?') from None

# Convenience access to common Engine attributes and methods

Expand Down
2 changes: 1 addition & 1 deletion spinach/contrib/spinachd/mail.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,6 @@ def serialize_email_messages(messages: List[EmailMessage]):
def deserialize_email_messages(messages: List[str]):
"""Deserialize EmailMessages passed as task argument."""
return [
pickle.loads(zlib.decompress(base64.b64decode(m)))
pickle.loads(zlib.decompress(base64.b64decode(m))) # noqa: S301
for m in messages
]
2 changes: 1 addition & 1 deletion spinach/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def check_signature(self):
msg = 'Arguments of job not compatible with task {}: {}'.format(
self.task_name, e
)
raise InvalidJobSignatureError(msg)
raise InvalidJobSignatureError(msg) from None
except ValueError:
logger.info('Cannot verify job signature, assuming it is correct')

Expand Down
1 change: 1 addition & 0 deletions spinach/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ class Tasks:
:arg max_concurrency: maximum number of simultaneous Jobs that can be
started for this Task. Requires max_retries to be also set.
"""

# This class is not thread-safe because it doesn't need to be used
# concurrently.

Expand Down
2 changes: 1 addition & 1 deletion spinach/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def exponential_backoff(attempt: int, cap: int=1200) -> timedelta:
"""
base = 3
temp = min(base * 2 ** attempt, cap) // 2
return timedelta(seconds=temp + random.randint(0, temp))
return timedelta(seconds=temp + random.randint(0, temp)) # noqa: S311


@contextlib.contextmanager
Expand Down
3 changes: 1 addition & 2 deletions spinach/worker.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from abc import ABC
import asyncio
from logging import getLogger
try:
Expand All @@ -18,7 +17,7 @@
logger = getLogger(__name__)


class BaseWorkers(ABC):
class BaseWorkers:
"""Base class for Spinach workers.
The Workers class receives jobs from the Engine via the `submit_job`
Expand Down
1 change: 0 additions & 1 deletion tests/functional/test_concurrency.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import pytest
import time

from spinach.brokers.memory import MemoryBroker
from spinach.brokers.redis import RedisBroker
Expand Down
2 changes: 1 addition & 1 deletion tests/test_memory_brokers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import pytest

from spinach.brokers.memory import MemoryBroker
from spinach.job import Job, JobStatus
from spinach.job import Job
from spinach.task import Task


Expand Down
4 changes: 2 additions & 2 deletions tests/test_redis_brokers.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def test_get_jobs_from_queue_returns_all_requested(broker):


def test_set_concurrency_keys_sets_new_keys_on_idempotent_tasks(broker):
max_c = random.randint(1, 10)
max_c = random.randint(1, 10) # noqa: S311
tasks = [
Task(
print, 'foo', 'q1', 10, timedelta(seconds=5),
Expand Down Expand Up @@ -229,7 +229,7 @@ def test_set_concurrency_keys_ignores_tasks_without_concurrency(broker):


def test_set_concurrency_keys_doesnt_overwrite_existing_concurrency(broker):
current_c = random.randint(1, 10)
current_c = random.randint(1, 10) # noqa: S311
set_concurrency_keys(broker, current_concurrency=current_c)
tasks = [
Task(
Expand Down
Loading

0 comments on commit 4f57896

Please sign in to comment.