ci: (partially) sync ci configuration to pymplate

- add python 3.12
- add ruff

.ci/release

@@ -19,10 +19,8 @@ from pathlib import Path
 from subprocess import check_call
 import shutil
 
-
 is_ci = os.environ.get('CI') is not None
 
-
 def main() -> None:
  import argparse
  p = argparse.ArgumentParser()
@@ -31,7 +29,7 @@ def main() -> None:
 
  extra = []
  if args.test:
- extra.extend(['--repository-url', 'https://test.pypi.org/legacy/'])
+ extra.extend(['--repository', 'testpypi'])
 
  root = Path(__file__).absolute().parent.parent
  os.chdir(root) # just in case
@@ -44,7 +42,7 @@ def main() -> None:
  if dist.exists():
  shutil.rmtree(dist)
 
- check_call('python3 setup.py sdist bdist_wheel', shell=True)
+ check_call(['python3', '-m', 'build'])
 
  TP = 'TWINE_PASSWORD'
  password = os.environ.get(TP)

.github/workflows/main.yml

@@ -9,24 +9,32 @@ on:
  pull_request: # needed to trigger on others' PRs
  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
  workflow_dispatch: # needed to trigger workflows manually
+ # todo cron?
+ inputs:
+ debug_enabled:
+ type: boolean
+ description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+ required: false
+ default: false
 
 
 jobs:
  build:
  strategy:
- # useful to quickly debug all versions, otherwise a bit wasteful
- # fail-fast: false
+ fail-fast: false
  matrix:
  platform: [ubuntu-latest, macos-latest, windows-latest]
- python-version: ['3.8', '3.9', '3.10', '3.11']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
  exclude: [
  # windows runners are pretty scarce, so let's only run lowest and highest python version
- {platform: windows-latest, python-version: '3.9'},
+ {platform: windows-latest, python-version: '3.9' },
  {platform: windows-latest, python-version: '3.10'},
+ {platform: windows-latest, python-version: '3.11'},
 
  # same, macos is a bit too slow and ubuntu covers python quirks well
  {platform: macos-latest , python-version: '3.9' },
- {platform: macos-latest , python-version: '3.10' },
+ {platform: macos-latest , python-version: '3.10'},
+ {platform: macos-latest , python-version: '3.11'},
  ]
 
  runs-on: ${{ matrix.platform }}
@@ -36,9 +44,6 @@ jobs:
  steps:
  # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
  - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
- - if: ${{ matrix.platform == 'macos-latest' && matrix.python-version == '3.11' }}
- # hmm somehow only seems necessary for 3.11 on osx??
- run: echo "$HOME/Library/Python/${{ matrix.python-version }}/bin" >> $GITHUB_PATH
 
  - uses: actions/setup-python@v4
  with:
@@ -47,9 +52,10 @@ jobs:
  - uses: actions/checkout@v3
  with:
  submodules: recursive
+ fetch-depth: 0 # nicer to have all git history when debugging/for tests
 
- # uncomment for SSH debugging
- # - uses: mxschmitt/action-tmate@v3
+ - uses: mxschmitt/action-tmate@v3
+  if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
 
  # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
  - run: bash .ci/run
@@ -128,16 +134,15 @@ jobs:
  if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
  env:
  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
- run: pip3 install --user wheel twine && .ci/release --test
- # TODO run pip install just to test?
+ run: pip3 install --user --upgrade build twine && .ci/release
 
  - name: 'release to pypi'
  # always deploy tags to release pypi
  # NOTE: release tags are guarded by on: push: tags on the top
  if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
  env:
  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
- run: pip3 install --user wheel twine && .ci/release
+ run: pip3 install --user --upgrade build twine && .ci/release
 
  ###
  build_extension:

doc/DEVELOPMENT.org

@@ -74,6 +74,7 @@ Example of running a test:
 
 You can also run them via tox:
 
+# TODO https://github.com/tox-dev/tox/discussions/2681
 : TOX_TESTENV_PASSENV='DISPLAY HOME' tox -e end2end -- -s -k 'test_blacklist_custom[firefox]'
 
 - it's useful to pass =--pdb= to pytest, so you don't have to restart webdriver all over

ruff.toml

@@ -0,0 +1,25 @@
+ignore = [
+### too opinionated style checks
+ "E501", # too long lines
+ "E702", # Multiple statements on one line (semicolon)
+ "E731", # assigning lambda instead of using def
+ "E741", # Ambiguous variable name: `l`
+ "E742", # Ambiguous class name: `O
+ "E401", # Multiple imports on one line
+ "F403", # import *` used; unable to detect undefined names
+###
+
+###
+ "E722", # Do not use bare `except` ## Sometimes it's useful for defensive imports and that sort of thing..
+ "F811", # Redefinition of unused # this gets in the way of pytest fixtures (e.g. in cachew)
+
+## might be nice .. but later and I don't wanna make it strict
+ "E402", # Module level import not at top of file
+
+### maybe consider these soon
+# sometimes it's useful to give a variable a name even if we don't use it as a documentation
+# on the other hand, often is a sign of error
+ "F841", # Local variable `count` is assigned to but never used
+ "F401", # imported but unused
+###
+]

setup.py

@@ -58,12 +58,11 @@ def main() -> None:
  'click', # confirmations for end2end test (might remove dependency)
 
  'pyautogui', # for keyboard automation during end2end tests
- ],
- 'linting': [
- 'pytest',
 
- 'mypy',
- 'lxml', # for coverage reports
+ 'ruff',
+
+ 'mypy',
+ 'lxml', # for coverage reports
  ],
  **{k[0]: v for k, v in DEPS_SOURCES.items()},
  'all': list(chain.from_iterable(DEPS_SOURCES.values())),

src/promnesia/common.py

@@ -554,7 +554,7 @@ def get_system_tz() -> pytz.BaseTzInfo:
  return pytz.timezone(zone)
  except Exception as e:
  logger.exception(e)
- logger.error(f"Unknown time zone %s. Falling back to UTC. Please report this as a bug!", zone)
+ logger.error("Unknown time zone %s. Falling back to UTC. Please report this as a bug!", zone)
  return pytz.utc
 
 # used in misc/install_server.py

src/promnesia/server.py

@@ -255,8 +255,10 @@ def visits(request: VisitsRequest) -> VisitsResponse:
  url=url,
  # odd, doesn't work just with: x or (y and z)
  where=lambda table, url: or_(
- table.c.norm_url == url, # exact match
- and_(table.c.context != None, table.c.norm_url.startswith(url, autoescape=True)) # + child visits, but only 'interesting' ones
+ # exact match
+ table.c.norm_url == url,
+ # + child visits, but only 'interesting' ones
+ and_(table.c.context != None, table.c.norm_url.startswith(url, autoescape=True)) # noqa: E711
  ),
  )
 

src/promnesia/sources/auto.py

@@ -26,14 +26,14 @@
 from ..config import use_cores
 
 
-from .filetypes import EUrl
+from .filetypes import EUrl, Ctx
 from .auto_obsidian import obsidian_replacer
 from .auto_logseq import logseq_replacer
 
 
 def _collect(thing, path: List[str], result: List[EUrl]) -> None:
  if isinstance(thing, str):
- ctx: Ctx = tuple(path) # type: ignore
+ ctx: Ctx = tuple(path)
  result.extend([EUrl(url=u, ctx=ctx) for u in extract_urls(thing)])
  elif isinstance(thing, list):
  path.append('[]')

src/promnesia/sources/reddit.py

@@ -95,7 +95,7 @@ def _from_submission(self, i: 'Submission') -> Results:
 
  def _from_upvote(self, i: 'Upvote') -> Results:
  locator = Loc.make(
- title=f'Reddit upvote',
+ title='Reddit upvote',
  href=i.url,
  )
  yield from self._from_common(i, locator=locator)

src/promnesia/sources/rss.py

@@ -23,6 +23,6 @@ def index() -> Results:
  yield Visit(
  url=feed.url,
  dt=feed.created_at or default_datetime,
- context=f'RSS subscription', # TODO use 'provider', etc?
+ context='RSS subscription', # TODO use 'provider', etc?
  locator=locator,
  )

src/promnesia/sources/signal.py

@@ -310,8 +310,8 @@ def connect_db(
  sql_cmds.extend(
  [
  f"ATTACH DATABASE '{decrypted_file}' AS plaintext KEY '';",
- f"SELECT sqlcipher_export('plaintext');",
- f"DETACH DATABASE plaintext;",
+ "SELECT sqlcipher_export('plaintext');",
+ "DETACH DATABASE plaintext;",
  ]
  )
  sql = "\n".join(sql_cmds)

tox.ini

@@ -1,20 +1,25 @@
 [tox]
-minversion = 3.5
+minversion = 3.21
 # relies on the correct version of Python installed
-envlist = tests,mypy-core,mypy-misc
+envlist = ruff,tests,mypy-core,mypy-misc
 # NOTE: we don't run end2end by default since it requires elaborate setup
 # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
 # hack to prevent .tox from crapping to the project directory
-toxworkdir={env:TOXWORKDIR_BASE:}{toxinidir}/.tox
+toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox
 
 [testenv]
+# TODO how to get package name from setuptools?
+package_name = "promnesia"
 passenv =
+# useful for tests to know they are running under ci
  CI
  CI_*
  CIRCLE*
- # respect user's cache dirs to prevent tox from crapping into project dir
- MYPY_CACHE_DIR
+# respect user's cache dirs to prevent tox from crapping into project dir
  PYTHONPYCACHEPREFIX
+ MYPY_CACHE_DIR
+ RUFF_CACHE_DIR
+#
  MY_CONFIG
  # by default we don't run browser tests to avoid confusing people when they run locally
  # but we want them on CI, so we allow to pass through the variable when we do want to run them
@@ -23,18 +28,24 @@ passenv =
  # todo ugh this is all so confusing... need to simplify
 
 
+[testenv:ruff]
+commands =
+ {envpython} -m pip install --use-pep517 -e .[testing]
+ {envpython} -m ruff src/
+
+
 [testenv:tests]
 commands =
- pip install -e .[testing]
+ {envpython} -m pip install -e .[testing]
 
  # run this first before we install any extra dependenceies
  {envpython} -m pytest tests/imports.py::test_imports_lazy
 
- pip install -e .[all]
- pip install -e .[HPI,org]
+ {envpython} -m pip install -e .[all]
+ {envpython} -m pip install -e .[HPI,org]
 
  # used in some tests
- hpi module install my.google.takeout.parser
+ {envpython} -m my.core module install my.google.takeout.parser
  {envpython} -m pytest tests {posargs}
 
 
@@ -43,45 +54,45 @@ setenv =
  WITH_BROWSER_TESTS=true
  PYTEST_TIMEOUT=120
 commands =
- pip install -e .[testing] hpi
+ {envpython} -m pip install -e .[testing] hpi
  {envpython} -m pytest tests/end2end_test.py {posargs}
 
 
 [testenv:mypy-core]
 commands =
- pip install -e .[linting]
+ {envpython} -m pip install -e .[testing]
  {envpython} -m mypy --install-types --non-interactive \
-  # note: sources are tested separately, below
-  -p promnesia --exclude 'sources/*' \
-  # txt report is a bit more convenient to view on CI
-  --txt-report .coverage.mypy-core \
-  --html-report .coverage.mypy-core \
-  {posargs}
+ # note: sources are tested separately, below
+ -p {[testenv]package_name} --exclude 'sources/*' \
+ # txt report is a bit more convenient to view on CI
+ --txt-report .coverage.mypy-core \
+ --html-report .coverage.mypy-core \
+ {posargs}
 
 
 [testenv:mypy-misc]
 commands =
- pip install -e .[linting]
- pip install -e .[HPI,org]
+ {envpython} -m pip install -e .[testing]
+ {envpython} -m pip install -e .[HPI,org]
  # todo install from HPI[all] or something?
 
- hpi module install my.github.ghexport
- hpi module install my.hypothesis
- hpi module install my.instapaper
- hpi module install my.pocket
- hpi module install my.reddit
- hpi module install my.fbmessenger
- hpi module install my.google.takeout.parser
- hpi module install my.browser.export
+ {envpython} -m my.core module install my.github.ghexport
+ {envpython} -m my.core module install my.hypothesis
+ {envpython} -m my.core module install my.instapaper
+ {envpython} -m my.core module install my.pocket
+ {envpython} -m my.core module install my.reddit
+ {envpython} -m my.core module install my.fbmessenger
+ {envpython} -m my.core module install my.google.takeout.parser
+ {envpython} -m my.core module install my.browser.export
 
  {envpython} -m mypy --install-types --non-interactive \
-  -p promnesia.sources \
-  # txt report is a bit more convenient to view on CI
-  --txt-report .coverage.mypy-misc \
-  --html-report .coverage.mypy-misc \
-  {posargs}
+ -p {[testenv]package_name}.sources \
+ # txt report is a bit more convenient to view on CI
+ --txt-report .coverage.mypy-misc \
+ --html-report .coverage.mypy-misc \
+ {posargs}
 
- pip install -e .[testing]
+ {envpython} -m pip install -e .[testing]
  # ugh. a bit crap to run it separately
  # but first will need to move tests inside the package if we want otherwise?
  # and I recall it was problematic at times..