opendatateam · bolinocroustibat · Jul 12, 2024 · Jul 15, 2024 · Jul 15, 2024 · Oct 4, 2022
diff --git a/.gitignore b/.gitignore
@@ -83,6 +83,9 @@ data
 # Pyenv
 .python-version
 
+# Package managers (Rye, Pipenv, Poetry...)
+requirements*.lock
+
 # Ruff
 .ruff_cache
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Current (in progress)
 
+- Switch to `pyproject.toml` [#2769](https://github.com/opendatateam/udata/pull/2769)
 - Add linter and formatter with `pyproject.toml` config, add lint and formatting step in CI, add pre-commit hook to lint and format, update docs and lint and format the code [#3085](https://github.com/opendatateam/udata/pull/3085)
 - Update pinned dependencies according to project dependencies, without updating any project dependencies [#3089](https://github.com/opendatateam/udata/pull/3089)
 - Add "run" button to harvesters (configurable with `HARVEST_ENABLE_MANUAL_RUN`) [#3092](https://github.com/opendatateam/udata/pull/3092)

diff --git a/docs/adapting-settings.md b/docs/adapting-settings.md
@@ -213,6 +213,16 @@ SEARCH_SERVICE_API_URL = 'http://127.0.0.1:5000/api/1/'
 ```
 
 See [udata-search-service][udata-search-service] for more information on using a search service.
+You'll need a Kakfa broker for the search service to work. See `KAFKA_URI`.
+
+## Kafka configuration
+
+### KAFKA_URI
+
+**default**: None
+
+Optionnal Kafka URI to use. If specified, messages can be consumed and produced in udata workers.
+It is required for an independent search service.
 
 ## Spatial configuration
 
@@ -252,12 +262,6 @@ HANDLED_LEVELS = ('fr:commune', 'fr:departement', 'fr:region')
 
 The number of items to fetch while previewing an harvest source
 
-### HARVEST_MAX_ITEMS
-
-**default**: `None`
-
-The max number of items to fetch when harvesting (development setting)
-
 ### HARVEST_DEFAULT_SCHEDULE
 
 **default**: `0 0 * * *`
@@ -451,9 +455,9 @@ You can see the full options list in
 
 ### CACHE_TYPE
 
-**default**: `'flask_caching.backends.redis'`
+**default**: `'redis'`
 
-The cache type, which can be adjusted to your needs (_ex:_ `null`, `flask_caching.backends.memcached`)
+The cache type, which can be adjusted to your needs (_ex:_ `null`, `memcached`)
 
 ### CACHE_KEY_PREFIX
 
@@ -549,6 +553,14 @@ Whether or not discussions should be enabled on posts
 
 The default page size for post listing
 
+## Datasets configuration
+
+### DATASET_MAX_RESOURCES_UNCOLLAPSED
+
+**default** `6`
+
+Max number of resources to display uncollapsed in dataset view.
+
 ## Sentry configuration
 
 ### SENTRY_DSN
@@ -558,7 +570,7 @@ The default page size for post listing
 The Sentry DSN associated to this udata instance.
 If defined, the Sentry support is automatically activated.
 
-`sentry-sdk[flask]` needs to be installed for this to work. This requirement is specified in `requirements/sentry.pip`.
+`sentry-sdk[flask]` needs to be installed for this to work. This requirement is specified in `pyproject.toml`.
 
 ### SENTRY_TAGS
 

diff --git a/docs/getting-started.md b/docs/getting-started.md
@@ -10,7 +10,7 @@ We’ll use the following repositories:
 # Check the system requirements
 
 !!! info
- Be aware that udata now requires Python **>3.9,<=3.11** to work. 
+ Be aware that udata now requires Python **>3.9,<=3.11** to work.
 
 udata requires several libraries to be installed to work. You can see them on the udata documentation link below.
 
@@ -27,7 +27,7 @@ $UDATA_WORKSPACE
 ├── fs
 ├── udata
 │ ├── ...
-│ ├── setup.py
+│ ├── pyproject.toml
 │ └── udata.cfg
 └── udata-front
  ├── ...
@@ -259,7 +259,7 @@ You can now visit `dev.local:7000/` in your browser and start playing with your
 You can use parcel to watch for file changes in udata or udata-front directory with
 
 ```bash
-inv assets-watch 
+inv assets-watch
 ```
 
 !!! note "Tell us what you think"

diff --git a/docs/harvesting.md b/docs/harvesting.md
@@ -26,7 +26,7 @@ def inner_harvest():
 def inner_process_dataset(item: HarvestItem, args1, args2, args3):
  dataset = self.get_dataset(item.remote_id)
 
- update_dataset(dataset, args1, args2) 
+ update_dataset(dataset, args1, args2)
 
  return dataset
 ```
@@ -225,18 +225,13 @@ class RandomBackend(BaseBackend):
 
 ```
 
-You need to properly expose the harvester as a `udata.harvesters` entrypoint in your `setup.py`:
+You need to properly expose the harvester as a `udata.harvesters` entrypoint in your `pyproject.toml`:
 
-```python
-setup(
- '...'
- entry_points={
- 'udata.harvesters': [
- 'random = canonical.path.to_the:RandomBackend',
- ]
- },
- '...'
-)
+```toml
+[project.entry-points."udata.harvesters"]
+csw-dcat = "udata.harvest.backends.dcat:CswDcatBackend"
+csw-iso-19139 = "udata.harvest.backends.dcat:CswIso19139DcatBackend"
+dcat = "udata.harvest.backends.dcat:DcatBackend"
 ```
 
 The easiest way is to start from the

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,192 @@
+[project]
+name = "udata"
+description = "Customizable and skinnable social platform dedicated to (open) data"
+keywords = ["udata opendata portal data"]
+authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Web Environment",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.11",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ "Topic :: System :: Software Distribution",
+]
+dependencies = [
+ "Babel==2.12.1",
+ "Faker==3.0.0",
+ "Flask==2.1.2",
+ "Flask-CDN==1.5.3",
+ "Flask-Caching==2.0.2",
+ "Flask-Cors==3.0.8",
+ "Flask-Gravatar==0.5.0",
+ "Flask-Login==0.6.2",
+ "Flask-Mail==0.9.1",
+ "Flask-Security-Too==5.1.2",
+ "Flask-Sitemap==0.4.0",
+ "Flask-WTF==1.0.1",
+ "Jinja2==3.1.2",
+ "StringDist==1.0.9",
+ "authlib==0.14.3",
+ "awesome-slugify==1.6.5",
+ "bcrypt==3.1.7",
+ "bleach==3.3.1",
+ "blinker==1.4",
+ "boto3>=1.26",
+ "celery==5.3.1",
+ "celerybeat-mongo==0.2.0",
+ "click==8.1.2",
+ "cryptography==42.0.8",
+ "factory-boy==2.12.0",
+ "flask-babel==4.0.0",
+ "flask-mongoengine==1.0.0",
+ "flask-navigation-temp-python-311==0.2.1",
+ "flask-restx==1.0.5",
+ "flask-storage==1.3.2",
+ "geojson==2.5.0",
+ "geomet==1.1.0",
+ "html2text==2019.9.26",
+ "kombu[redis]==5.3.1",
+ "langdetect==1.0.9",
+ "lxml==4.9.3",
+ "mistune==0.8.4",
+ "mongoengine==0.27.0",
+ "netaddr==0.7.19",
+ "pydenticon==0.3.1",
+ "pymongo==4.3.3",
+ "python-dateutil==2.8.2",
+ "pytz==2024.1",
+ "rdflib==6.0.0",
+ "redis==4.5.2",
+ "requests==2.24.0",
+ "speaklater==1.3",
+ "tlds",
+ "urlextract==0.14.0",
+ "voluptuous==0.11.7",
+ "werkzeug==2.2.2",
+ "wtforms==3.0.1",
+ "wtforms-json==0.3.5",
+]
+readme = "README.md"
+requires-python = ">= 3.9"
+license = { file = "LICENSE" }
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/opendatateam/udata"
+
+[project.scripts]
+udata = "udata.commands:cli"
+
+[project.entry-points]
+[project.entry-points.pytest11]
+udata = "udata.tests.plugin"
+
+[project.entry-points."udata.avatars"]
+adorable = "udata.features.identicon.backends:adorable"
+internal = "udata.features.identicon.backends:internal"
+robohash = "udata.features.identicon.backends:robohash"
+
+[project.entry-points."udata.harvesters"]
+csw-dcat = "udata.harvest.backends.dcat:CswDcatBackend"
+csw-iso-19139 = "udata.harvest.backends.dcat:CswIso19139DcatBackend"
+dcat = "udata.harvest.backends.dcat:DcatBackend"
+
+[build-system]
+requires = ["setuptools>=61.2"]
+build-backend = "setuptools.build_meta"
+
+[tool.distutils.compile_catalog]
+domain = "udata"
+directory = "udata/translations"
+statistics = true
+
+[tool.distutils.extract_messages]
+keywords = "_ N_:1,2 P_:1c,2 L_ gettext ngettext:1,2 pgettext:1c,2 npgettext:1c,2,3 lazy_gettext lazy_pgettext:1c,2"
+mapping-file = "babel.cfg"
+add-comments = "TRANSLATORS:"
+output-file = "udata/translations/udata.pot"
+width = 80
+
+[tool.distutils.init_catalog]
+domain = "udata"
+input-file = "udata/translations/udata.pot"
+output-dir = "udata/translations"
+
+[tool.distutils.update_catalog]
+domain = "udata"
+input-file = "udata/translations/udata.pot"
+output-dir = "udata/translations"
+previous = true
+
+[tool.pytest.ini_options]
+norecursedirs = [
+ ".git",
+ "build",
+ ".tox",
+ "specs",
+ ".cache",
+ "udata/static",
+ "udata/templates",
+ "udata/translations",
+]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+python_classes = ["*Test"]
+env = """
+# See: https://docs.authlib.org/en/latest/flask/oauth2.html
+AUTHLIB_INSECURE_TRANSPORT=true"""
+mock_use_standalone_module = true
+# See: https://docs.pytest.org/en/latest/warnings.html#deprecationwarning-and-pendingdeprecationwarning
+filterwarnings = ["ignore::DeprecationWarning:mongoengine"]
+# See: https://docs.pytest.org/en/latest/example/markers.html#registering-markers
+markers = [
+ "frontend: load the frontend stack",
+ "preview: mock a preview backend",
+ "oauth: inject an OAuth client",
+]
+
+[tool.ruff]
+lint = { select = ["I"] } # also sort imports with an isort rule
+line-length = 100
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+ "readme-renderer[md]>=44.0",
+ "invoke>=2.2.0",
+ "pytest-cov>=5.0.0",
+]
+
+[tool.setuptools]
+include-package-data = false
+packages = ["udata", "tasks"]
+
+# Tox (http://tox.testrun.org/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tool.tox]
+legacy_tox_ini = """
+ [tox]
+ envlist = py37, doc
+
+ [testenv]
+ commands =
+ python setup.py develop
+ nosetests udata
+ install_command = pip3 install --allow-external webassets --allow-unverified webassets --pre {opts} {packages}
+ deps =
+ -r{toxinidir}/requirements/install.pip
+ -r{toxinidir}/requirements/test.pip
+
+ [testenv:doc]
+ changedir = doc
+ deps = sphinx
+ commands = sphinx-build -b html -d {envtmpdir}/doctrees . {envtmpdir}/html
+"""
diff --git a/requirements/install.pip b/requirements/install.pip
@@ -73,7 +73,7 @@ click-plugins==1.1.1
  # via celery
 click-repl==0.3.0
  # via celery
-cryptography==2.8
+cryptography==42.0.8
  # via
  # -r requirements/install.in
  # authlib

diff --git a/ruff.toml b/ruff.toml
diff --git a/setup.cfg b/setup.cfg