Add a CKAN client

.github/workflows/pytest.yaml

@@ -43,11 +43,14 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-        cache: pip
-        cache-dependency-path: "**/pyproject.toml"
+
+    - uses: astral-sh/setup-uv@v4
+      with:
+        enable-cache: true
+        cache-dependency-glob: "**/pyproject.toml"
 
     - name: Install the Python package and dependencies
-      run: pip install --upgrade --upgrade-strategy=eager .[tests]
+      run: uv pip install --system --upgrade .[tests]
 
     - name: Clone the transport-data/registry repo
       run: coverage run -m transport_data store clone

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.2
+  rev: v1.13.0
   hooks:
   - id: mypy
     additional_dependencies:
@@ -20,7 +20,7 @@ repos:
     - types-tqdm
     args: []
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.9
+  rev: v0.8.1
   hooks:
   - id: ruff
   - id: ruff-format

doc/whatsnew.rst

@@ -1,8 +1,11 @@
 What's new
 **********
 
-.. Next release
-.. ============
+Next release
+============
+
+- Add :mod:`.util.ckan`, including a :class:`~.ckan.Client` for access to CKAN instances (including the TDC instances); proxy classes for CKAN objects including :class:`~.ckan.Package`, :class:`~.ckan.Resource`, and mode (:pull:`35`).
+- Add :data:`.org.ckan.PROD` and :data:`.org.ckan.DEV` instances of :class:`.ckan.Client` (:pull:`35`).
 
 v24.11.25
 =========

pyproject.toml

@@ -22,6 +22,7 @@ classifiers = [
 readme = "README.rst"
 requires-python = ">=3.9"
 dependencies = [
+  "ckanapi",
   "click",
   "docutils",
   "dsss[git-store] >= 1.3.0",
@@ -72,6 +73,7 @@ exclude = ["build/"]
 
 [[tool.mypy.overrides]]
 module = [
+  "ckanapi.*",
   "google_auth_oauthlib.*",
   "google.*",
   "googleapiclient.*",

transport_data/adb/__init__.py

@@ -36,22 +36,49 @@
 #: Mapping from short codes for ATO data categories to file names.
 FILES = {
     # "ATO National Database Masterlist of Indicators",
-    "ACC": "ATO Workbook (ACCESS & CONNECTIVITY (ACC)).xlsx",
-    "APH": "ATO Workbook (AIR POLLUTION & HEALTH (APH)).xlsx",
-    "CLC": "ATO Workbook (CLIMATE CHANGE (CLC)).xlsx",
-    "INF": "ATO Workbook (INFRASTRUCTURE (INF)).xlsx",
-    "MIS": "ATO Workbook (MISCELLANEOUS (MIS)).xlsx",
-    "POL": "ATO Workbook (TRANSPORT POLICY (POL)).xlsx",
-    "RSA": "ATO Workbook (ROAD SAFETY (RSA)).xlsx",
-    "SEC": "ATO Workbook (SOCIO-ECONOMIC (SEC)).xlsx",
-    "TAS": "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS)).xlsx",
+    "ACC": (
+        "ATO Workbook (ACCESS & CONNECTIVITY (ACC)).xlsx",
+        "sha256:21c3b7e662d932f5cc61c22489acb3cf0e8a70200abc2372c7fe212602903fd7",
+    ),
+    "APH": (
+        "ATO Workbook (AIR POLLUTION & HEALTH (APH)).xlsx",
+        "sha256:b06c102a1184ed83d77673146599e11c2b6c81d784ebcee6b46f4d43713e899a",
+    ),
+    "CLC": (
+        "ATO Workbook (CLIMATE CHANGE (CLC)).xlsx",
+        "sha256:7fe2d4bb656508bf3194406d25ed04c1ac403daaaf1ada74847a2c330efcfb2a",
+    ),
+    "INF": (
+        "ATO Workbook (INFRASTRUCTURE (INF)).xlsx",
+        "sha256:30b9d05330838809ff0d53b2e61ade935eccdb4b73c0509fd1fc49b405699ac3",
+    ),
+    "MIS": (
+        "ATO Workbook (MISCELLANEOUS (MIS)).xlsx",
+        "sha256:2ef3cdc5e6363cdca1f671bbf12bf0463fe8a4210cb49b3d32ebd2440c6fe6df",
+    ),
+    "POL": (
+        "ATO Workbook (TRANSPORT POLICY (POL)).xlsx",
+        "sha256:fbf23b012590b631239654d255d23ccb70fa717b466be8343a5b0f1e8b4ce720",
+    ),
+    "RSA": (
+        "ATO Workbook (ROAD SAFETY (RSA)).xlsx",
+        "sha256:a4285d129c2739c8660a07a5d1c9902ec21c3cd4d13a2a9dfe9d49daca2c0dd5",
+    ),
+    "SEC": (
+        "ATO Workbook (SOCIO-ECONOMIC (SEC)).xlsx",
+        "sha256:b5d2ee5d07b5554ef262436ef898afd976fbb4956bd7cb850829cb7391d207c0",
+    ),
+    "TAS": (
+        "ATO Workbook (TRANSPORT ACTIVITY & SERVICES (TAS)).xlsx",
+        "sha256:628d4e9774f84d30d80706e982e4ddf7187d77f8676e69765c307701da1caf77",
+    ),
 }
 
 VERSION = "0.1.0"
 
 
 def expand(fname: str) -> str:
-    return FILES.get(fname, fname)
+    return FILES.get(fname, (fname, None))[0]
 
 
 def _make_url(fname: str) -> str:
@@ -61,7 +88,7 @@ def _make_url(fname: str) -> str:
 POOCH = Pooch(
     module=__name__,
     base_url=BASE_URL,
-    registry={expand(key): None for key in FILES.keys()},
+    registry={v[0]: v[1] for v in FILES.values()},
     urls={expand(key): _make_url(key) for key in FILES.keys()},
     expand=expand,
 )

transport_data/data/tests/ckan/package.json

@@ -0,0 +1,116 @@
+{
+  "author": null,
+  "author_email": null,
+  "contributors": [],
+  "creator_user_id": "50bb4200-615b-4921-a8da-58f9d53ad9d0",
+  "data_access": "publicly available",
+  "data_provider": "OICA",
+  "dimensioning": "vehicle production by country/region and type (passenger cars, LDV, heavy trucks, buses & coaches)",
+  "frequency": "as_needed",
+  "geographies": [],
+  "groups": [
+    {
+      "description": "",
+      "display_name": "Worldwide",
+      "id": "85cb14f3-72ca-47a7-b8f0-ad5fc78de45a",
+      "image_display_url": "",
+      "name": "worldwide",
+      "title": "Worldwide"
+    }
+  ],
+  "id": "8decd067-a38a-4834-a70c-40419b17fe9c",
+  "indicators": [
+    "Vehicle production"
+  ],
+  "is_archived": false,
+  "isopen": true,
+  "language": "en",
+  "license_id": "CC-BY-4.0",
+  "license_title": "Creative Commons Attribution 4.0",
+  "license_url": "https://creativecommons.org/licenses/by/4.0/",
+  "maintainer": null,
+  "maintainer_email": null,
+  "metadata_created": "2024-10-29T12:25:58.933848",
+  "metadata_modified": "2024-10-29T12:26:02.452514",
+  "modes": [
+    "cars",
+    "private-cars",
+    "truck",
+    "bus"
+  ],
+  "name": "2023-production-statistics",
+  "notes": "This OICA statistics web page contains world motor vehicle production statistics, obtained from national trade organizations, OICA members or correspondents.",
+  "num_resources": 1,
+  "num_tags": 0,
+  "organization": {
+    "approval_status": "approved",
+    "created": "2024-10-29T12:13:41.185874",
+    "description": "",
+    "id": "5eb6e69f-0b49-4c16-8cfa-3fdbba4492e7",
+    "image_url": "",
+    "is_organization": true,
+    "name": "oica",
+    "state": "active",
+    "title": "OICA",
+    "type": "organization"
+  },
+  "owner_org": "5eb6e69f-0b49-4c16-8cfa-3fdbba4492e7",
+  "private": false,
+  "regions": [
+    "worldwide"
+  ],
+  "relationships_as_object": [],
+  "relationships_as_subject": [],
+  "resources": [
+    {
+      "cache_last_updated": null,
+      "cache_url": null,
+      "created": "2024-10-29T12:26:02.463411",
+      "datastore_active": false,
+      "description": null,
+      "format": "",
+      "hash": "",
+      "id": "e28f2207-e153-451e-a9e2-234473c50897",
+      "last_modified": null,
+      "metadata_modified": "2024-10-29T12:26:02.458704",
+      "mimetype": null,
+      "mimetype_inner": null,
+      "name": "Passenger-Cars-2023",
+      "package_id": "8decd067-a38a-4834-a70c-40419b17fe9c",
+      "position": 0,
+      "resource_type": "data",
+      "size": null,
+      "state": "active",
+      "url": "https://www.oica.net/wp-content/uploads/Passenger-Cars-2023.xlsx",
+      "url_type": null
+    }
+  ],
+  "sectors": [
+    "road"
+  ],
+  "services": [
+    "passenger"
+  ],
+  "sources": [
+    {
+      "title": "Cooperation with Ward Auto (America)",
+      "url": "https://www.wardsauto.com/"
+    },
+    {
+      "title": "Asian Automotive Analysis Fourin (Asia)",
+      "url": "https://aaa.fourin.com/"
+    }
+  ],
+  "state": "active",
+  "tags": [],
+  "tdc_category": "public",
+  "temporal_coverage_end": "2023-01-01",
+  "temporal_coverage_start": "1999-01-01",
+  "title": "2023 production statistics",
+  "type": "dataset",
+  "units": [
+    "#"
+  ],
+  "url": "https://www.oica.net/production-statistics/",
+  "version": null
+}

transport_data/oica/__init__.py

@@ -140,7 +140,7 @@ def convert_single_file(
         pattern = r"(PC|CV|TOTAL) WORLD VEHICLES IN USE +\(in thousand units\)"
         units = "kvehicle"
 
-    if match := re.fullmatch(pattern, df.iloc[0, 0]):
+    if match := re.fullmatch(pattern, str(df.iloc[0, 0])):
         # Codes are inconsistent across files; use shorter codes
         vehicle_type = {
             "ALL TYPES": "_T",
@@ -192,7 +192,7 @@ def convert_single_file(
     for m, group_df in df.groupby("MEASURE"):
         # Create structures for this measure
         # TODO Retrieve possibly-cached or -stored structures
-        dfd, dsd = get_structures(m)
+        dfd, dsd = get_structures(str(m))
 
         # Create a data set
         ds = DataSet(described_by=dfd, structured_by=dsd)

transport_data/org/ckan.py

@@ -0,0 +1,12 @@
+""":class:`.ckan.Client` objects to access TDC instances."""
+
+from transport_data.util.ckan import Client
+
+#: Development instance of TDC CKAN. Accessible as of 2024-11-27.
+DEV = Client("https://ckan.tdc.dev.datopian.com")
+
+#: Production instance of TDC CKAN. Accessible as of 2024-11-27.
+PROD = Client("https://ckan.transport-data.org")
+
+#: Staging instance of TDC CKAN. Not accessible as of 2024-11-27; SSL certificate error.
+STAGING = Client("https://ckan.tdc.staging.datopian.com")

transport_data/org/metadata/spreadsheet.py

@@ -2,14 +2,15 @@
 
 import logging
 import re
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple, cast
 
 from sdmx.model import common, v21
 
 if TYPE_CHECKING:
     import pathlib
 
     from openpyxl import Workbook
+    from openpyxl.cell.cell import Cell
     from openpyxl.worksheet.worksheet import Worksheet
 
     from transport_data.util.sdmx import MAKeywords
@@ -57,7 +58,7 @@
 def _header(ws: "Worksheet", *columns: Tuple[str, int]) -> None:
     """Write header columns and format their style and width."""
     for column, (value, width) in enumerate(columns, start=1):
-        cell = ws.cell(row=1, column=column, value=value)
+        cell = cast("Cell", ws.cell(row=1, column=column, value=value))
         cell.style = "header"
         ws.column_dimensions[cell.column_letter].width = width
 

transport_data/tests/org/test_ckan.py

@@ -0,0 +1,26 @@
+import pytest
+from requests.exceptions import SSLError
+
+from transport_data.org.ckan import DEV, PROD, STAGING
+
+
+@pytest.mark.parametrize(
+    "instance, N_exp",
+    [
+        # (DEV, 451),
+        (DEV, 5),
+        # (PROD, 418),
+        (PROD, 5),
+        pytest.param(STAGING, None, marks=pytest.mark.xfail(raises=SSLError)),
+    ],
+)
+def test_main(instance, N_exp: int) -> None:
+    result = instance.package_list(max=5)
+
+    assert N_exp <= len(result)
+
+    p = instance.package_show(result[0])
+
+    assert p is result[0]  # Same object instance returned
+
+    assert "dataset" == p.type