From e518d704a0a65458d53ad8e9b85897d54c22529b Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Fri, 12 Jul 2024 11:25:09 +0200 Subject: [PATCH 01/12] feat: use full pyproject.toml --- pyproject.toml | 77 ++++++++++++++++++++++++++++++++++++++++ requirements/install.pip | 2 +- ruff.toml | 4 --- setup.cfg | 43 ---------------------- setup.py | 70 ------------------------------------ 5 files changed, 78 insertions(+), 118 deletions(-) create mode 100644 pyproject.toml delete mode 100644 ruff.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e51d91bd7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[project] +name = "udata" +version = "2.0.4" +description = "Customizable and skinnable social platform dedicated to (open) data" +authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }] +dependencies = [ + "Babel==2.12.1", + "Faker==3.0.0", + "Flask==2.1.2", + "Flask-CDN==1.5.3", + "Flask-Caching==2.0.2", + "Flask-Cors==3.0.8", + "Flask-Gravatar==0.5.0", + "Flask-Login==0.6.2", + "Flask-Mail==0.9.1", + "Flask-Security-Too==5.1.2", + "Flask-Sitemap==0.4.0", + "Flask-WTF==1.0.1", + "Jinja2==3.1.2", + "StringDist==1.0.9", + "authlib==0.14.3", + "awesome-slugify==1.6.5", + "bcrypt==3.1.7", + "bleach==3.3.1", + "blinker==1.4", + "boto3>=1.26", + "celery==5.3.1", + "celerybeat-mongo==0.2.0", + "click==8.1.2", + "cryptography==42.0.8", + "factory-boy==2.12.0", + "flask-babel==4.0.0", + "flask-mongoengine==1.0.0", + "flask-navigation-temp-python-311==0.2.1", + "flask-restx==1.0.5", + "flask-storage==1.3.2", + "geojson==2.5.0", + "geomet==1.1.0", + "html2text==2019.9.26", + "kombu[redis]==5.3.1", + "langdetect==1.0.9", + "lxml==4.9.3", + "mistune==0.8.4", + "mongoengine==0.27.0", + "netaddr==0.7.19", + "pydenticon==0.3.1", + "pymongo==4.3.3", + "python-dateutil==2.8.2", + "pytz==2024.1", + "rdflib==6.0.0", + "redis==4.5.2", + "requests==2.24.0", + "speaklater==1.3", + "tlds", + "urlextract==0.14.0", + "voluptuous==0.11.7", + "werkzeug==2.2.2", + "wtforms==3.0.1", + "wtforms-json==0.3.5", +] +readme = "README.md" +requires-python = ">= 3.11" +license = { file = "LICENSE" } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/udata"] diff --git a/requirements/install.pip b/requirements/install.pip index 26dcdb180..96bff6357 100644 --- a/requirements/install.pip +++ b/requirements/install.pip @@ -73,7 +73,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -cryptography==2.8 +cryptography==42.0.8 # via # -r requirements/install.in # authlib diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 5594e9f32..000000000 --- a/ruff.toml +++ /dev/null @@ -1,4 +0,0 @@ -line-length = 100 - -[lint] -select = ["I"] # also sort imports with an isort rule diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9b33406ba..000000000 --- a/setup.cfg +++ /dev/null @@ -1,43 +0,0 @@ -[compile_catalog] -domain = udata -directory = udata/translations -statistics = true - -[extract_messages] -keywords = _ N_:1,2 P_:1c,2 L_ gettext ngettext:1,2 pgettext:1c,2 npgettext:1c,2,3 lazy_gettext lazy_pgettext:1c,2 -mapping_file = babel.cfg -add_comments = TRANSLATORS: -output_file = udata/translations/udata.pot -width = 80 - -[init_catalog] -domain = udata -input_file = udata/translations/udata.pot -output_dir = udata/translations - -[update_catalog] -domain = udata -input_file = udata/translations/udata.pot -output_dir = udata/translations -previous = true - -[tool:pytest] -norecursedirs = .git build .tox specs .cache udata/static udata/templates udata/translations -python_files = test_*.py -python_functions = test_* -python_classes = *Test -env = - # See: https://docs.authlib.org/en/latest/flask/oauth2.html - AUTHLIB_INSECURE_TRANSPORT=true -mock_use_standalone_module = true -# See: https://docs.pytest.org/en/latest/warnings.html#deprecationwarning-and-pendingdeprecationwarning -filterwarnings = - ignore::DeprecationWarning:mongoengine -# See: https://docs.pytest.org/en/latest/example/markers.html#registering-markers -markers = - frontend: load the frontend stack - preview: mock a preview backend - oauth: inject an OAuth client - -[wheel] -universal = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 87989f0af..000000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -import os - -from setuptools import find_packages, setup - - -def file_content(filename): - """Load file content""" - with open(filename) as ifile: - return ifile.read() - - -def pip(filename): - """Return path to pip requirements file""" - return file_content(os.path.join("requirements", filename)) - - -long_description = "\n".join((file_content("README.md"), file_content("CHANGELOG.md"), "")) - - -install_requires = pip("install.pip") - -setup( - name="udata", - version=__import__("udata").__version__, - description=__import__("udata").__description__, - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/opendatateam/udata", - author="Opendata Team", - author_email="opendatateam@data.gouv.fr", - packages=find_packages(), - include_package_data=True, - python_requires=">=3.8", - install_requires=install_requires, - entry_points={ - "console_scripts": [ - "udata = udata.commands:cli", - ], - "udata.harvesters": [ - "dcat = udata.harvest.backends.dcat:DcatBackend", - "csw-dcat = udata.harvest.backends.dcat:CswDcatBackend", - "csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend", - ], - "udata.avatars": [ - "internal = udata.features.identicon.backends:internal", - "adorable = udata.features.identicon.backends:adorable", - "robohash = udata.features.identicon.backends:robohash", - ], - "pytest11": [ - "udata = udata.tests.plugin", - ], - }, - license="GNU AGPLv3+", - keywords="udata opendata portal data", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python", - "Environment :: Web Environment", - "Operating System :: OS Independent", - "Intended Audience :: Developers", - "Topic :: System :: Software Distribution", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Topic :: Software Development :: Libraries :: Python Modules", - ("License :: OSI Approved :: GNU Affero General Public License v3" " or later (AGPLv3+)"), - ], -) From 6ea3d4f07a4711980985901863fe7c9406f46890 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 17:33:52 +0200 Subject: [PATCH 02/12] build: use pyproject instead of setup.cfg for build config --- pyproject.toml | 88 ++++++++++++++++++++++++++++++++++++++++++++++---- setup.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index e51d91bd7..990ce332f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,17 +61,93 @@ dependencies = [ readme = "README.md" requires-python = ">= 3.11" license = { file = "LICENSE" } +dynamic = ["version"] [build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[tool.distutils.compile_catalog] +domain = "udata" +directory = "udata/translations" +statistics = true + +[tool.distutils.extract_messages] +keywords = "_ N_:1,2 P_:1c,2 L_ gettext ngettext:1,2 pgettext:1c,2 npgettext:1c,2,3 lazy_gettext lazy_pgettext:1c,2" +mapping-file = "babel.cfg" +add-comments = "TRANSLATORS:" +output-file = "udata/translations/udata.pot" +width = 80 + +[tool.distutils.init_catalog] +domain = "udata" +input-file = "udata/translations/udata.pot" +output-dir = "udata/translations" + +[tool.distutils.update_catalog] +domain = "udata" +input-file = "udata/translations/udata.pot" +output-dir = "udata/translations" +previous = true + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "build", + ".tox", + "specs", + ".cache", + "udata/static", + "udata/templates", + "udata/translations", +] +python_files = ["test_*.py"] +python_functions = ["test_*"] +python_classes = ["*Test"] +env = """ +# See: https://docs.authlib.org/en/latest/flask/oauth2.html +AUTHLIB_INSECURE_TRANSPORT=true""" +mock_use_standalone_module = true +# See: https://docs.pytest.org/en/latest/warnings.html#deprecationwarning-and-pendingdeprecationwarning +filterwarnings = ["ignore::DeprecationWarning:mongoengine"] +# See: https://docs.pytest.org/en/latest/example/markers.html#registering-markers +markers = [ + "frontend: load the frontend stack", + "preview: mock a preview backend", + "oauth: inject an OAuth client", +] + +[tool.flake8] +max-line-length = "100" +exclude = """ +.cache, +.git, +__pycache__, +build, +data, +dist, +docs, +instance, +js, +less, +node_modules, +reports, +requirements, +specs, +udata/static, +udata/templates""" + + +[tool.ruff] +lint = { select = ["I"] } # also sort imports with an isort rule +line-length = 100 [tool.rye] managed = true dev-dependencies = [] -[tool.hatch.metadata] -allow-direct-references = true +[tool.setuptools] +include-package-data = false -[tool.hatch.build.targets.wheel] -packages = ["src/udata"] +[tool.wheel] +universal = "1" diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..19eaed69d --- /dev/null +++ b/setup.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +import os + +from setuptools import setup, find_packages + + +def file_content(filename): + '''Load file content''' + with open(filename) as ifile: + return ifile.read() + + +def pip(filename): + """Return path to pip requirements file""" + return file_content(os.path.join('requirements', filename)) + + +long_description = '\n'.join(( + file_content('README.md'), + file_content('CHANGELOG.md'), + '' +)) + + +install_requires = pip('install.pip') + +setup( + name='udata', + version=__import__('udata').__version__, + description=__import__('udata').__description__, + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/opendatateam/udata', + author='Opendata Team', + author_email='opendatateam@data.gouv.fr', + packages=find_packages(), + include_package_data=True, + python_requires='>=3.8', + install_requires=install_requires, + entry_points={ + 'console_scripts': [ + 'udata = udata.commands:cli', + ], + 'udata.harvesters': [ + 'dcat = udata.harvest.backends.dcat:DcatBackend', + 'csw-dcat = udata.harvest.backends.dcat:CswDcatBackend', + 'csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend' + ], + 'udata.avatars': [ + 'internal = udata.features.identicon.backends:internal', + 'adorable = udata.features.identicon.backends:adorable', + 'robohash = udata.features.identicon.backends:robohash', + ], + 'pytest11': [ + 'udata = udata.tests.plugin', + ], + }, + license='GNU AGPLv3+', + keywords='udata opendata portal data', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Programming Language :: Python', + 'Environment :: Web Environment', + 'Operating System :: OS Independent', + 'Intended Audience :: Developers', + 'Topic :: System :: Software Distribution', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Topic :: Software Development :: Libraries :: Python Modules', + ('License :: OSI Approved :: GNU Affero General Public License v3' + ' or later (AGPLv3+)'), + ], +) From cd18a8e01c7250e27f30f63c7137a786fda5b095 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 17:35:47 +0200 Subject: [PATCH 03/12] build: remove flake8 config --- pyproject.toml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 990ce332f..8c23449a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,27 +117,6 @@ markers = [ "oauth: inject an OAuth client", ] -[tool.flake8] -max-line-length = "100" -exclude = """ -.cache, -.git, -__pycache__, -build, -data, -dist, -docs, -instance, -js, -less, -node_modules, -reports, -requirements, -specs, -udata/static, -udata/templates""" - - [tool.ruff] lint = { select = ["I"] } # also sort imports with an isort rule line-length = 100 From dd31a045dcea133b0cbc36bd1e0bb959d166bb97 Mon Sep 17 00:00:00 2001 From: quaxsze Date: Tue, 4 Oct 2022 17:06:31 +0200 Subject: [PATCH 04/12] docs: update changelog # Conflicts: # CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e4aa146..6ec58982b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Current (in progress) +- Switch to `pyproject.toml` [#2769](https://github.com/opendatateam/udata/pull/2769) - Add linter and formatter with `pyproject.toml` config, add lint and formatting step in CI, add pre-commit hook to lint and format, update docs and lint and format the code [#3085](https://github.com/opendatateam/udata/pull/3085) - Update pinned dependencies according to project dependencies, without updating any project dependencies [#3089](https://github.com/opendatateam/udata/pull/3089) - Add "run" button to harvesters (configurable with `HARVEST_ENABLE_MANUAL_RUN`) [#3092](https://github.com/opendatateam/udata/pull/3092) From 6eff21c67d7ec0298ec00eff3e45cfab850bf36e Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:02:14 +0200 Subject: [PATCH 05/12] fix: fix pyproject.toml --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8c23449a1..b8b3d3dbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,4 @@ dev-dependencies = [] [tool.setuptools] include-package-data = false - -[tool.wheel] -universal = "1" +packages = ["udata", "tasks"] From 6f0578bef0ca3798670eb13d1fab1b49f5e444f1 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:16:06 +0200 Subject: [PATCH 06/12] style: lint setup.py --- setup.py | 81 ++++++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index 19eaed69d..eb797276e 100644 --- a/setup.py +++ b/setup.py @@ -2,74 +2,69 @@ import os -from setuptools import setup, find_packages +from setuptools import find_packages, setup def file_content(filename): - '''Load file content''' + """Load file content""" with open(filename) as ifile: return ifile.read() def pip(filename): """Return path to pip requirements file""" - return file_content(os.path.join('requirements', filename)) + return file_content(os.path.join("requirements", filename)) -long_description = '\n'.join(( - file_content('README.md'), - file_content('CHANGELOG.md'), - '' -)) +long_description = "\n".join((file_content("README.md"), file_content("CHANGELOG.md"), "")) -install_requires = pip('install.pip') +install_requires = pip("install.pip") setup( - name='udata', - version=__import__('udata').__version__, - description=__import__('udata').__description__, + name="udata", + version=__import__("udata").__version__, + description=__import__("udata").__description__, long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/opendatateam/udata', - author='Opendata Team', - author_email='opendatateam@data.gouv.fr', + long_description_content_type="text/markdown", + url="https://github.com/opendatateam/udata", + author="Opendata Team", + author_email="opendatateam@data.gouv.fr", packages=find_packages(), include_package_data=True, - python_requires='>=3.8', + python_requires=">=3.9", install_requires=install_requires, entry_points={ - 'console_scripts': [ - 'udata = udata.commands:cli', + "console_scripts": [ + "udata = udata.commands:cli", ], - 'udata.harvesters': [ - 'dcat = udata.harvest.backends.dcat:DcatBackend', - 'csw-dcat = udata.harvest.backends.dcat:CswDcatBackend', - 'csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend' + "udata.harvesters": [ + "dcat = udata.harvest.backends.dcat:DcatBackend", + "csw-dcat = udata.harvest.backends.dcat:CswDcatBackend", + "csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend", ], - 'udata.avatars': [ - 'internal = udata.features.identicon.backends:internal', - 'adorable = udata.features.identicon.backends:adorable', - 'robohash = udata.features.identicon.backends:robohash', + "udata.avatars": [ + "internal = udata.features.identicon.backends:internal", + "adorable = udata.features.identicon.backends:adorable", + "robohash = udata.features.identicon.backends:robohash", ], - 'pytest11': [ - 'udata = udata.tests.plugin', + "pytest11": [ + "udata = udata.tests.plugin", ], }, - license='GNU AGPLv3+', - keywords='udata opendata portal data', + license="GNU AGPLv3+", + keywords="udata opendata portal data", classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python', - 'Environment :: Web Environment', - 'Operating System :: OS Independent', - 'Intended Audience :: Developers', - 'Topic :: System :: Software Distribution', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Topic :: Software Development :: Libraries :: Python Modules', - ('License :: OSI Approved :: GNU Affero General Public License v3' - ' or later (AGPLv3+)'), + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python", + "Environment :: Web Environment", + "Operating System :: OS Independent", + "Intended Audience :: Developers", + "Topic :: System :: Software Distribution", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Topic :: Software Development :: Libraries :: Python Modules", + ("License :: OSI Approved :: GNU Affero General Public License v3" " or later (AGPLv3+)"), ], ) From 8a5da6d2ffd249f51c640e6446058ae7f8a5bc08 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:16:20 +0200 Subject: [PATCH 07/12] fix: fix python version in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b8b3d3dbf..bd996a089 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ dependencies = [ "wtforms-json==0.3.5", ] readme = "README.md" -requires-python = ">= 3.11" +requires-python = ">= 3.9" license = { file = "LICENSE" } dynamic = ["version"] From 69d11056c37fe41e9ffad7996928fcb886f551e9 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:22:46 +0200 Subject: [PATCH 08/12] build: fix pyproject.toml --- pyproject.toml | 35 ++++++++++++++++++++++++- setup.py | 70 -------------------------------------------------- 2 files changed, 34 insertions(+), 71 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index bd996a089..ed381f38a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,21 @@ [project] name = "udata" -version = "2.0.4" description = "Customizable and skinnable social platform dedicated to (open) data" +keywords = ["udata opendata portal data"] authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Software Distribution", +] dependencies = [ "Babel==2.12.1", "Faker==3.0.0", @@ -63,6 +76,26 @@ requires-python = ">= 3.9" license = { file = "LICENSE" } dynamic = ["version"] +[project.urls] +Homepage = "https://github.com/opendatateam/udata" + +[project.scripts] +udata = "udata.commands:cli" + +[project.entry-points] +[project.entry-points.pytest11] +udata = "udata.tests.plugin" + +[project.entry-points."udata.avatars"] +adorable = "udata.features.identicon.backends:adorable" +internal = "udata.features.identicon.backends:internal" +robohash = "udata.features.identicon.backends:robohash" + +[project.entry-points."udata.harvesters"] +csw-dcat = "udata.harvest.backends.dcat:CswDcatBackend" +csw-iso-19139 = "udata.harvest.backends.dcat:CswIso19139DcatBackend" +dcat = "udata.harvest.backends.dcat:DcatBackend" + [build-system] requires = ["setuptools>=61.2"] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py deleted file mode 100644 index eb797276e..000000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python - -import os - -from setuptools import find_packages, setup - - -def file_content(filename): - """Load file content""" - with open(filename) as ifile: - return ifile.read() - - -def pip(filename): - """Return path to pip requirements file""" - return file_content(os.path.join("requirements", filename)) - - -long_description = "\n".join((file_content("README.md"), file_content("CHANGELOG.md"), "")) - - -install_requires = pip("install.pip") - -setup( - name="udata", - version=__import__("udata").__version__, - description=__import__("udata").__description__, - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/opendatateam/udata", - author="Opendata Team", - author_email="opendatateam@data.gouv.fr", - packages=find_packages(), - include_package_data=True, - python_requires=">=3.9", - install_requires=install_requires, - entry_points={ - "console_scripts": [ - "udata = udata.commands:cli", - ], - "udata.harvesters": [ - "dcat = udata.harvest.backends.dcat:DcatBackend", - "csw-dcat = udata.harvest.backends.dcat:CswDcatBackend", - "csw-iso-19139 = udata.harvest.backends.dcat:CswIso19139DcatBackend", - ], - "udata.avatars": [ - "internal = udata.features.identicon.backends:internal", - "adorable = udata.features.identicon.backends:adorable", - "robohash = udata.features.identicon.backends:robohash", - ], - "pytest11": [ - "udata = udata.tests.plugin", - ], - }, - license="GNU AGPLv3+", - keywords="udata opendata portal data", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python", - "Environment :: Web Environment", - "Operating System :: OS Independent", - "Intended Audience :: Developers", - "Topic :: System :: Software Distribution", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Topic :: Software Development :: Libraries :: Python Modules", - ("License :: OSI Approved :: GNU Affero General Public License v3" " or later (AGPLv3+)"), - ], -) From b520b2bcf25244d8209960937c5d3975962f50ec Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:35:58 +0200 Subject: [PATCH 09/12] docs: update docs --- docs/adapting-settings.md | 30 +++++++++++++++++++++--------- docs/getting-started.md | 6 +++--- docs/harvesting.md | 19 +++++++------------ 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/docs/adapting-settings.md b/docs/adapting-settings.md index 2aedcfbd8..d02fd6a53 100644 --- a/docs/adapting-settings.md +++ b/docs/adapting-settings.md @@ -213,6 +213,16 @@ SEARCH_SERVICE_API_URL = 'http://127.0.0.1:5000/api/1/' ``` See [udata-search-service][udata-search-service] for more information on using a search service. +You'll need a Kakfa broker for the search service to work. See `KAFKA_URI`. + +## Kafka configuration + +### KAFKA_URI + +**default**: None + +Optionnal Kafka URI to use. If specified, messages can be consumed and produced in udata workers. +It is required for an independent search service. ## Spatial configuration @@ -252,12 +262,6 @@ HANDLED_LEVELS = ('fr:commune', 'fr:departement', 'fr:region') The number of items to fetch while previewing an harvest source -### HARVEST_MAX_ITEMS - -**default**: `None` - -The max number of items to fetch when harvesting (development setting) - ### HARVEST_DEFAULT_SCHEDULE **default**: `0 0 * * *` @@ -451,9 +455,9 @@ You can see the full options list in ### CACHE_TYPE -**default**: `'flask_caching.backends.redis'` +**default**: `'redis'` -The cache type, which can be adjusted to your needs (_ex:_ `null`, `flask_caching.backends.memcached`) +The cache type, which can be adjusted to your needs (_ex:_ `null`, `memcached`) ### CACHE_KEY_PREFIX @@ -549,6 +553,14 @@ Whether or not discussions should be enabled on posts The default page size for post listing +## Datasets configuration + +### DATASET_MAX_RESOURCES_UNCOLLAPSED + +**default** `6` + +Max number of resources to display uncollapsed in dataset view. + ## Sentry configuration ### SENTRY_DSN @@ -558,7 +570,7 @@ The default page size for post listing The Sentry DSN associated to this udata instance. If defined, the Sentry support is automatically activated. -`sentry-sdk[flask]` needs to be installed for this to work. This requirement is specified in `requirements/sentry.pip`. +`sentry-sdk[flask]` needs to be installed for this to work. This requirement is specified in `pyproject.toml`. ### SENTRY_TAGS diff --git a/docs/getting-started.md b/docs/getting-started.md index 84f82b12b..582cb97fb 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -10,7 +10,7 @@ We’ll use the following repositories: # Check the system requirements !!! info - Be aware that udata now requires Python **>3.9,<=3.11** to work. + Be aware that udata now requires Python **>3.9,<=3.11** to work. udata requires several libraries to be installed to work. You can see them on the udata documentation link below. @@ -27,7 +27,7 @@ $UDATA_WORKSPACE ├── fs ├── udata │ ├── ... -│ ├── setup.py +│ ├── pyproject.toml │ └── udata.cfg └── udata-front ├── ... @@ -259,7 +259,7 @@ You can now visit `dev.local:7000/` in your browser and start playing with your You can use parcel to watch for file changes in udata or udata-front directory with ```bash -inv assets-watch +inv assets-watch ``` !!! note "Tell us what you think" diff --git a/docs/harvesting.md b/docs/harvesting.md index e19e66f8b..cb9e7cb99 100644 --- a/docs/harvesting.md +++ b/docs/harvesting.md @@ -26,7 +26,7 @@ def inner_harvest(): def inner_process_dataset(item: HarvestItem, args1, args2, args3): dataset = self.get_dataset(item.remote_id) - update_dataset(dataset, args1, args2) + update_dataset(dataset, args1, args2) return dataset ``` @@ -225,18 +225,13 @@ class RandomBackend(BaseBackend): ``` -You need to properly expose the harvester as a `udata.harvesters` entrypoint in your `setup.py`: +You need to properly expose the harvester as a `udata.harvesters` entrypoint in your `pyproject.toml`: -```python -setup( - '...' - entry_points={ - 'udata.harvesters': [ - 'random = canonical.path.to_the:RandomBackend', - ] - }, - '...' -) +```toml +[project.entry-points."udata.harvesters"] +csw-dcat = "udata.harvest.backends.dcat:CswDcatBackend" +csw-iso-19139 = "udata.harvest.backends.dcat:CswIso19139DcatBackend" +dcat = "udata.harvest.backends.dcat:DcatBackend" ``` The easiest way is to start from the From 876462c1b77943a3c63bcc571db4672d6df34490 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:36:49 +0200 Subject: [PATCH 10/12] chore: update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 35f7ef735..167a82cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,9 @@ data # Pyenv .python-version +# Package managers (Rye, Pipenv, Poetry...) +requirements*.lock + # Ruff .ruff_cache From 7148568f4dba786a11ff75f19f82111cc34f9106 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:36:59 +0200 Subject: [PATCH 11/12] build: add dev dependencies --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed381f38a..065683e91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Software Distribution", ] @@ -156,7 +156,11 @@ line-length = 100 [tool.rye] managed = true -dev-dependencies = [] +dev-dependencies = [ + "readme-renderer[md]>=44.0", + "invoke>=2.2.0", + "pytest-cov>=5.0.0", +] [tool.setuptools] include-package-data = false From 4a59f3e2b560500d98f277e559d64e3c691bbe70 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Mon, 15 Jul 2024 19:46:48 +0200 Subject: [PATCH 12/12] build: move tox config to pyproject.tml --- pyproject.toml | 25 +++++++++++++++++++++++++ tox.ini | 20 -------------------- 2 files changed, 25 insertions(+), 20 deletions(-) delete mode 100644 tox.ini diff --git a/pyproject.toml b/pyproject.toml index 065683e91..3c8c56e85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,3 +165,28 @@ dev-dependencies = [ [tool.setuptools] include-package-data = false packages = ["udata", "tasks"] + +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tool.tox] +legacy_tox_ini = """ + [tox] + envlist = py37, doc + + [testenv] + commands = + python setup.py develop + nosetests udata + install_command = pip3 install --allow-external webassets --allow-unverified webassets --pre {opts} {packages} + deps = + -r{toxinidir}/requirements/install.pip + -r{toxinidir}/requirements/test.pip + + [testenv:doc] + changedir = doc + deps = sphinx + commands = sphinx-build -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +""" diff --git a/tox.ini b/tox.ini deleted file mode 100644 index dc37bd234..000000000 --- a/tox.ini +++ /dev/null @@ -1,20 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -envlist = py36, doc - -[testenv] -commands = - python setup.py develop - nosetests udata -deps = - -r{toxinidir}/requirements/install.pip - -r{toxinidir}/requirements/test.pip - -[testenv:doc] -changedir = doc -deps = sphinx -commands = sphinx-build -b html -d {envtmpdir}/doctrees . {envtmpdir}/html