From a2125ab67f5f2676797ca63edcd51425c08b3bca Mon Sep 17 00:00:00 2001 From: Friedrich Lindenberg Date: Tue, 21 Mar 2023 09:57:41 +0100 Subject: [PATCH] handle typed babel library (#1049) * handle typed babel library * expect state.locale to be correctly typed * use .parse * mama mia * install python in venv --- Dockerfile | 5 ++++- followthemoney/types/common.py | 4 ++-- followthemoney/types/country.py | 2 +- followthemoney/types/gender.py | 2 +- followthemoney/types/language.py | 2 +- followthemoney/types/topic.py | 2 +- followthemoney/util.py | 12 +++++++----- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1978d767..440d4c9e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ ENV DEBIAN_FRONTEND noninteractive # build-essential RUN apt-get -qq -y update \ && apt-get -qq -y install locales ca-certificates tzdata curl \ - python3-pip libpq-dev python3-icu python3-psycopg2 python3-cryptography \ + python3-pip libpq-dev python3-icu python3-psycopg2 \ + python3-venv python3-cryptography \ && apt-get -qq -y autoremove \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -23,6 +24,8 @@ RUN localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 ENV LANG="en_US.UTF-8" \ TZ="UTC" +RUN python3 -m venv /venv +ENV PATH="/venv/bin:$PATH" RUN pip3 install -q --no-cache-dir -U pip setuptools six psycopg2-binary COPY . /opt/followthemoney RUN pip3 install -q --no-cache-dir -e /opt/followthemoney diff --git a/followthemoney/types/common.py b/followthemoney/types/common.py index dd9e8e000..ef399a158 100644 --- a/followthemoney/types/common.py +++ b/followthemoney/types/common.py @@ -1,5 +1,5 @@ from itertools import product -from babel.core import Locale # type: ignore +from babel.core import Locale from banal import ensure_list from normality import stringify from typing import Any, Dict, Optional, Sequence, Callable, TYPE_CHECKING, TypedDict @@ -217,7 +217,7 @@ def __init__(self) -> None: self._names: Dict[Locale, EnumValues] = {} self.codes = set(self.names.keys()) - def _locale_names(self, locale: str) -> EnumValues: + def _locale_names(self, locale: Locale) -> EnumValues: return {} @property diff --git a/followthemoney/types/country.py b/followthemoney/types/country.py index 41c3ed8b3..4699f0a97 100644 --- a/followthemoney/types/country.py +++ b/followthemoney/types/country.py @@ -1,6 +1,6 @@ import countrynames from typing import Optional, TYPE_CHECKING -from babel.core import Locale # type: ignore +from babel.core import Locale from followthemoney.rdf import URIRef, Identifier from followthemoney.types.common import EnumType, EnumValues diff --git a/followthemoney/types/gender.py b/followthemoney/types/gender.py index 133d0ed48..066015df7 100644 --- a/followthemoney/types/gender.py +++ b/followthemoney/types/gender.py @@ -1,5 +1,5 @@ from typing import Optional, TYPE_CHECKING -from babel.core import Locale # type: ignore +from babel.core import Locale from followthemoney.types.common import EnumType, EnumValues from followthemoney.rdf import URIRef, Identifier diff --git a/followthemoney/types/language.py b/followthemoney/types/language.py index 74993f86b..a896e4dcc 100644 --- a/followthemoney/types/language.py +++ b/followthemoney/types/language.py @@ -1,5 +1,5 @@ from typing import Optional, TYPE_CHECKING -from babel.core import Locale # type: ignore +from babel.core import Locale from languagecodes import iso_639_alpha3 from followthemoney.types.common import EnumType, EnumValues diff --git a/followthemoney/types/topic.py b/followthemoney/types/topic.py index 70a5da65a..c21d4bdde 100644 --- a/followthemoney/types/topic.py +++ b/followthemoney/types/topic.py @@ -1,4 +1,4 @@ -from babel.core import Locale # type: ignore +from babel.core import Locale from followthemoney.types.common import EnumType, EnumValues from followthemoney.rdf import URIRef, Identifier diff --git a/followthemoney/util.py b/followthemoney/util.py index b5e11efb8..0bf984a1c 100644 --- a/followthemoney/util.py +++ b/followthemoney/util.py @@ -1,7 +1,7 @@ import os import logging from hashlib import sha1 -from babel import Locale # type: ignore +from babel import Locale from gettext import translation from threading import local @@ -31,7 +31,7 @@ def gettext(*args: Optional[str], **kwargs: Dict[str, str]) -> str: if not hasattr(state, "translation"): - set_model_locale(DEFAULT_LOCALE) + set_model_locale(Locale.parse(DEFAULT_LOCALE)) return cast(str, state.translation.gettext(*args, **kwargs)) @@ -42,14 +42,14 @@ def defer(text: str) -> str: def set_model_locale(locale: Locale) -> None: state.locale = locale state.translation = translation( - "followthemoney", i18n_path, [locale], fallback=True + "followthemoney", i18n_path, [str(locale)], fallback=True ) def get_locale() -> Locale: if not hasattr(state, "locale"): - return Locale(DEFAULT_LOCALE) - return Locale(state.locale) + return Locale.parse(DEFAULT_LOCALE) + return cast(Locale, state.locale) def get_env_list(name: str, default: List[str] = []) -> List[str]: @@ -138,6 +138,8 @@ def merge_context(left: Dict[K, V], right: Dict[K, V]) -> Dict[K, List[V]]: combined = {} keys = [*left.keys(), *right.keys()] for key in set(keys): + if key in ("caption",): + continue lval: List[V] = [i for i in ensure_list(left.get(key)) if i is not None] rval: List[V] = [i for i in ensure_list(right.get(key)) if i is not None] combined[key] = unique_list([*lval, *rval])