diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml index 8848370..4946756 100644 --- a/.github/workflows/Publish.yml +++ b/.github/workflows/Publish.yml @@ -15,19 +15,42 @@ jobs: - name: Retrieve source code uses: actions/checkout@v4 - - name: Build and publish Docker Image + - name: Build and publish Docker Image for UI uses: openzim/docker-publish-action@v10 with: - image-name: openzim/zimit-ui + image-name: openzim/zimit-frontend-ui on-master: latest restrict-to: openzim/zimit-frontend registries: ghcr.io - credentials: GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + dockerfile: Dockerfile-ui + # prettier-ignore + credentials: + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} - - name: Deploy Zimit frontend changes to zimit.kiwix.org + - name: Build and publish Docker Image for API + uses: openzim/docker-publish-action@v10 + with: + image-name: openzim/zimit-frontend-api + on-master: latest + restrict-to: openzim/zimit-frontend + registries: ghcr.io + dockerfile: Dockerfile-api + # prettier-ignore + credentials: + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + + - name: Deploy Zimit frontend UI changes to zimit.kiwix.org + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.ZIMIT_KUBE_CONFIG }} + with: + args: rollout restart deployments frontend-ui-deployment -n zimit + + - name: Deploy Zimit frontend API changes to zimit.kiwix.org uses: actions-hub/kubectl@master env: KUBE_CONFIG: ${{ secrets.ZIMIT_KUBE_CONFIG }} with: - args: rollout restart deployments ui-deployment -n zimit + args: rollout restart deployments frontend-api-deployment -n zimit diff --git a/.github/workflows/QA.yml b/.github/workflows/QA.yml index 51c175c..804f15f 100644 --- a/.github/workflows/QA.yml +++ b/.github/workflows/QA.yml @@ -63,3 +63,8 @@ jobs: working-directory: ui run: | yarn lint + + - name: Check Typescript typing + working-directory: ui + run: | + yarn type-check diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 59545ea..60bcc35 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -40,13 +40,24 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Ensure we can build the Docker image + - name: Ensure we can build the Docker image for UI run: | - docker build -t zimitfrontend . + docker build -t zimitfrontend-ui . -f Dockerfile-ui - - name: Ensure we can start the Docker image + - name: Ensure we can start the Docker image for UI run: | - docker run -d --rm --name test_container zimitfrontend + docker run -d --rm --name test_container zimitfrontend-ui + sleep 5 + docker ps | grep test_container + docker stop test_container + + - name: Ensure we can build the Docker image for API + run: | + docker build -t zimitfrontend-api . -f Dockerfile-api + + - name: Ensure we can start the Docker image for API + run: | + docker run -d --rm --name test_container zimitfrontend-api sleep 5 docker ps | grep test_container docker stop test_container diff --git a/Dockerfile b/Dockerfile-api similarity index 59% rename from Dockerfile rename to Dockerfile-api index 59abe5d..85de0c7 100644 --- a/Dockerfile +++ b/Dockerfile-api @@ -1,21 +1,6 @@ -FROM node:20-alpine as ui_builder - -RUN apk --no-cache add yarn -WORKDIR /src/ui -COPY ui/package.json ui/yarn.lock /src/ui/ -RUN yarn install -COPY ui/index.html /src/ui/ -COPY ui/*.json /src/ui/ -COPY ui/*.ts /src/ui/ -COPY ui/*.js /src/ui/ -COPY ui/public /src/ui/public -COPY ui/src /src/ui/src -COPY locales /src/locales -RUN NODE_ENV=production yarn build - FROM python:3.12-alpine -LABEL org.opencontainers.image.source https://github.com/offspot/metrics +LABEL org.opencontainers.image.source https://github.com/openzim/zimit-frontend # Specifying a workdir which is not "/"" is mandatory for proper uvicorn watchfiles # operation (used mostly only in dev, but changing the workdir does not harm production) @@ -25,11 +10,10 @@ WORKDIR "/home" RUN python -m pip install --no-cache-dir -U \ pip -# to set to your clients' origins +# Set to your client origin(s) ENV ALLOWED_ORIGINS http://localhost:8001|http://127.0.0.1:8001 -ENV DATABASE_URL sqlite+pysqlite:////data/database.db -ENV LOGWATCHER_DATA_FOLDER /data/logwatcher +# Copy minimal files for installation of project dependencies COPY api/pyproject.toml api/README.md /src/ COPY api/src/zimitfrontend/__about__.py /src/src/zimitfrontend/__about__.py @@ -44,8 +28,8 @@ COPY api/*.md /src/ RUN pip install --no-cache-dir /src \ && rm -rf /src +ENV LOCALES_LOCATION /locales COPY locales /locales -COPY --from=ui_builder /src/ui/dist /src/ui EXPOSE 80 diff --git a/Dockerfile-ui b/Dockerfile-ui new file mode 100644 index 0000000..b56e33a --- /dev/null +++ b/Dockerfile-ui @@ -0,0 +1,22 @@ +FROM node:20-alpine as ui_builder + +RUN apk --no-cache add yarn +WORKDIR /src/ui +COPY ui/package.json ui/yarn.lock /src/ui/ +RUN yarn install +COPY ui/index.html /src/ui/ +COPY ui/*.json /src/ui/ +COPY ui/*.ts /src/ui/ +COPY ui/*.js /src/ui/ +COPY ui/public /src/ui/public +COPY ui/src /src/ui/src +COPY locales /src/locales +RUN NODE_ENV=production yarn build + + +FROM caddy:2.8-alpine +LABEL org.opencontainers.image.source https://github.com/openzim/zimit-frontend + +COPY --from=ui_builder /src/ui/dist /usr/share/caddy + +COPY locales /locales diff --git a/README.md b/README.md index ef46431..cabe725 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Docker](https://ghcr-badge.deta.dev/openzim/zimit-ui/latest_tag?label=docker)](https://ghcr.io/openzim/zimit-ui) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) -This project is a UI (and it API / backend-for-frontend) allowing any user to submit +This project is a UI (and its API / backend-for-frontend) allowing any user to submit Zimit requests to a Zimfarm instance. It is NOT a standalone tool allowing to run zimit scraper. A Zimfarm instance and associated worker(s) is required for the system to be functional. diff --git a/api/pyproject.toml b/api/pyproject.toml index 30edb8b..b0b8d9a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -204,5 +204,5 @@ include = ["src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**", "src/zimitfrontend/templates", ".hatch"] extraPaths = ["src"] pythonVersion = "3.12" -typeCheckingMode = "basic" +typeCheckingMode = "strict" disableBytesTypePromotions = true diff --git a/api/src/zimitfrontend/constants.py b/api/src/zimitfrontend/constants.py index 179bb61..f88a627 100644 --- a/api/src/zimitfrontend/constants.py +++ b/api/src/zimitfrontend/constants.py @@ -20,7 +20,7 @@ def _get_int_setting(environment_variable_name: str, default_value: int) -> int: """Get environment variable as integer or fallback to default value""" try: - return int(os.getenv(environment_variable_name, default_value)) + return int(os.getenv(environment_variable_name) or default_value) except Exception as exc: logger.error( f"Unable to parse {environment_variable_name}: " @@ -34,7 +34,7 @@ def _get_size_setting(environment_variable_name: str, default_value: str) -> int """Get environment variable as size (parsed with unit) or fallback to default""" try: return humanfriendly.parse_size( - os.getenv(environment_variable_name, default_value) + os.getenv(environment_variable_name) or default_value ) except Exception as exc: logger.error( @@ -45,6 +45,24 @@ def _get_size_setting(environment_variable_name: str, default_value: str) -> int return humanfriendly.parse_size(default_value) +def _get_time_setting(environment_variable_name: str, default_value: str) -> float: + """Get environment variable as time (parsed with unit) or fallback to default + + Returned value is in seconds, not matter the unit passed in environement variable + """ + try: + return humanfriendly.parse_timespan( + os.getenv(environment_variable_name) or default_value + ) + except Exception as exc: + logger.error( + f"Unable to apply custom {environment_variable_name}: " + f"{os.getenv(environment_variable_name)}. " + f"Using {default_value}. Error: {exc}" + ) + return humanfriendly.parse_timespan(default_value) + + class ApiConfiguration: """Shared backend configuration""" @@ -56,7 +74,7 @@ class ApiConfiguration: zimfarm_api_url = os.getenv( "INTERNAL_ZIMFARM_WEBAPI", "https://api.farm.zimit.kiwix.org/v1" ) - zimfarm_requests_timeout = int(os.getenv("ZIMFARM_REQUESTS_TIMEOUT", "10")) + zimfarm_requests_timeout = _get_time_setting("ZIMFARM_REQUESTS_TIMEOUT", "10s") mailgun_requests_timeout = int(os.getenv("MAILGUN_REQUESTS_TIMEOUT", "10")) zimfarm_username = os.getenv("_ZIMFARM_USERNAME", "-") zimfarm_password = os.getenv("_ZIMFARM_PASSWORD", "-") @@ -73,7 +91,7 @@ class ApiConfiguration: # mailgun mailgun_from = os.getenv("MAILGUN_FROM", "Zimit ") - mailgun_api_key = os.getenv("MAILGUN_API_KEY", "") + mailgun_api_key = os.getenv("MAILGUN_API_KEY") mailgun_api_url = os.getenv( "MAILGUN_API_URL", "https://api.mailgun.net/v3/mg.zimit.kiwix.org" ) @@ -91,7 +109,7 @@ class ApiConfiguration: ) hook_token = os.getenv("HOOK_TOKEN", uuid.uuid4().hex) - ui_location = pathlib.Path(os.getenv("UI_LOCATION", "/src/ui")) + locales_location = pathlib.Path(os.getenv("LOCALES_LOCATION", "../locales")) # list of rtl language codes rtl_language_codes = ("fa",) diff --git a/api/src/zimitfrontend/i18n.py b/api/src/zimitfrontend/i18n.py index 91f6a8b..0e740d9 100644 --- a/api/src/zimitfrontend/i18n.py +++ b/api/src/zimitfrontend/i18n.py @@ -1,34 +1,37 @@ from pathlib import Path +from typing import Any -import i18n +import i18n # pyright: ignore -from zimitfrontend.constants import logger +from zimitfrontend.constants import ApiConfiguration, logger -def setup_i18n(): +def setup_i18n() -> None: """Configure python-i18n""" - i18n.set("locale", "en") - i18n.set("fallback", "en") - i18n.set("file_format", "json") - i18n.set("filename_format", "{locale}.{format}") - i18n.set("skip_locale_root_data", True) - - prod_folder = Path("/locales") - if prod_folder.exists(): - logger.info(f"Loading locales from {prod_folder}") - i18n.load_path.append(prod_folder) - - dev_folder = Path(Path(__file__).parent.parent.parent.parent / "locales") - if dev_folder.exists(): - logger.info(f"Loading locales from {dev_folder}") - i18n.load_path.append(dev_folder) - - -def set_locale(lang): + i18n.set("locale", "en") # pyright: ignore[reportUnknownMemberType] + i18n.set("fallback", "en") # pyright: ignore[reportUnknownMemberType] + i18n.set("file_format", "json") # pyright: ignore[reportUnknownMemberType] + i18n.set( # pyright: ignore[reportUnknownMemberType] + "filename_format", "{locale}.{format}" + ) + i18n.set("skip_locale_root_data", True) # pyright: ignore[reportUnknownMemberType] + + locales_location = Path(ApiConfiguration.locales_location) + if not locales_location.exists(): + raise Exception(f"Missing locales folder '{locales_location}'") + logger.info(f"Loading locales from {locales_location}") + i18n.load_path.append(locales_location) # pyright: ignore + + +def change_locale(lang: str) -> None: """Change locale""" - i18n.set("locale", lang) + i18n.set("locale", lang) # pyright: ignore[reportUnknownMemberType] -def t(key, **kwargs): +def t(key: str, **kwargs: Any) -> str: """Get translated string""" - return i18n.t(key, **kwargs) + return ( + i18n.t( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + key, **kwargs + ) + ) diff --git a/api/src/zimitfrontend/main.py b/api/src/zimitfrontend/main.py index ee231f6..f424eeb 100644 --- a/api/src/zimitfrontend/main.py +++ b/api/src/zimitfrontend/main.py @@ -1,11 +1,9 @@ from http import HTTPStatus -from pathlib import Path from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, JSONResponse, RedirectResponse -from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from fastapi.responses import JSONResponse, RedirectResponse from starlette.requests import Request from zimitfrontend import __about__ @@ -23,6 +21,7 @@ def create_app(self) -> FastAPI: ) @self.app.get("/api") + @self.app.get("/") async def landing() -> RedirectResponse: # pyright: ignore """Redirect to root of latest version of the API""" return RedirectResponse( @@ -53,7 +52,9 @@ async def landing() -> RedirectResponse: # pyright: ignore ) @api.exception_handler(500) - async def internal_exception_handler(_: Request, exc: Exception): + async def internal_exception_handler( # pyright: ignore[reportUnusedFunction] + _: Request, exc: Exception + ): logger.exception( exc ) # log the exception which occured so that we can debug @@ -75,52 +76,4 @@ async def internal_exception_handler(_: Request, exc: Exception): self.app.mount(f"/api/{__about__.__api_version__}", api) - class ServeVueUiFromRoot(BaseHTTPMiddleware): - """Custom middleware to serve the Vue.JS application - - We need a bit of black magic to: - - serve the Vue.JS UI from "/" - - but still keep the API on "/api" - - and support Vue.JS routes like "/home" - - and still return 404 when the UI is requesting a file which does not exits - """ - - ui_location = Path() - - async def dispatch( - self, request: Request, call_next: RequestResponseEndpoint - ): - path = request.url.path - - # API is served normally - if path.startswith("/api"): - response = await call_next(request) - return response - - # Serve index.html on root - if path == "/": - return FileResponse( - ApiConfiguration.ui_location.joinpath("index.html") - ) - - local_path = ApiConfiguration.ui_location.joinpath(path[1:]) - - # If there is no dot, then we are probably serving a Vue.JS internal - # route, so let's serve Vue.JS app - if "." not in local_path.name: - return FileResponse( - ApiConfiguration.ui_location.joinpath("index.html") - ) - - # If the path exists and is a file, serve it - if local_path.exists() and local_path.is_file(): - return FileResponse(local_path) - - # Otherwise continue to next handler (which is probably a 404) - response = await call_next(request) - return response - - # Apply the custom middleware - self.app.add_middleware(ServeVueUiFromRoot) - return self.app diff --git a/api/src/zimitfrontend/routes/hook.py b/api/src/zimitfrontend/routes/hook.py index 34b490f..760a7d6 100644 --- a/api/src/zimitfrontend/routes/hook.py +++ b/api/src/zimitfrontend/routes/hook.py @@ -33,7 +33,8 @@ def webhook( if result.status == SUCCESS and result.target and result.subject and result.body: try: resp = send_email_via_mailgun(result.target, result.subject, result.body) - logger.info(f"Mailgun notif sent: {resp}") + if resp: + logger.info(f"Mailgun notif sent: {resp}") except Exception as exc: logger.error(f"Failed to send mailgun notif: {exc}", exc_info=exc) return result.status diff --git a/api/src/zimitfrontend/routes/requests.py b/api/src/zimitfrontend/routes/requests.py index f119b29..fa98b81 100644 --- a/api/src/zimitfrontend/routes/requests.py +++ b/api/src/zimitfrontend/routes/requests.py @@ -7,7 +7,7 @@ from zimitfrontend.constants import ApiConfiguration, logger from zimitfrontend.routes.schemas import TaskCreateRequest, TaskCreateResponse, TaskInfo -from zimitfrontend.routes.utils import convert_zimfarm_task_to_info +from zimitfrontend.routes.utils import get_task_info from zimitfrontend.zimfarm import query_api router = APIRouter( @@ -29,12 +29,10 @@ def task_info( task_id: Annotated[str, Path()], ) -> TaskInfo: # first try to find the task - success, status, task = query_api("GET", f"/tasks/{task_id}?hide_secrets=") + _, status, task = query_api("GET", f"/tasks/{task_id}?hide_secrets=") if status == HTTPStatus.NOT_FOUND: # if it fails, try to find the requested task - success, status, task = query_api( - "GET", f"/requested-tasks/{task_id}?hide_secrets=" - ) + _, status, task = query_api("GET", f"/requested-tasks/{task_id}?hide_secrets=") if status != HTTPStatus.OK: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, @@ -43,7 +41,7 @@ def task_info( "zimfarm_message": task, }, ) - return convert_zimfarm_task_to_info(task) + return get_task_info(task) @router.post( @@ -114,7 +112,7 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int: } # create schedule payload - payload = { + payload = { # pyright: ignore[reportUnknownVariableType] "name": schedule_name, "language": {"code": "eng", "name_en": "English", "name_native": "English"}, "category": "other", @@ -132,7 +130,7 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int: f"&target={request.email}" f"&lang={request.lang}" ) - payload.update( + payload.update( # pyright: ignore[reportUnknownMemberType] { "notification": { "requested": {"webhook": [webhook_url]}, @@ -142,7 +140,11 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int: ) # create a unique schedule for that request on the zimfarm - success, status, resp = query_api("POST", "/schedules/", payload=payload) + success, status, resp = query_api( + "POST", + "/schedules/", + payload=payload, # pyright: ignore[reportUnknownArgumentType] + ) if not success: logger.error(f"Unable to create schedule via HTTP {status}: {resp}") message = f"Unable to create schedule via HTTP {status}: {resp}" diff --git a/api/src/zimitfrontend/routes/utils.py b/api/src/zimitfrontend/routes/utils.py index b4809d4..d36117a 100644 --- a/api/src/zimitfrontend/routes/utils.py +++ b/api/src/zimitfrontend/routes/utils.py @@ -1,7 +1,7 @@ from typing import Any from zimitfrontend.constants import ApiConfiguration, logger -from zimitfrontend.i18n import set_locale +from zimitfrontend.i18n import change_locale from zimitfrontend.routes.schemas import ( HookStatus, MailToSend, @@ -15,7 +15,7 @@ SUCCESS = HookStatus(status="success") -def convert_zimfarm_task_to_info(task: Any) -> TaskInfo: +def get_task_info(task: Any) -> TaskInfo: """Transforms a task object(dict) returned by Zimfarm API The final object is ready to be consumed by the frontend, with most checks for @@ -121,7 +121,7 @@ def convert_hook_to_mail( } logger.info(f"Translating to {lang}") - set_locale(lang) + change_locale(lang) subject = ( jinja_env.get_template("email_subject.txt").render(**context).replace("\n", "") ) diff --git a/api/src/zimitfrontend/templates/email_body.html b/api/src/zimitfrontend/templates/email_body.html index e8a6697..53e49d8 100644 --- a/api/src/zimitfrontend/templates/email_body.html +++ b/api/src/zimitfrontend/templates/email_body.html @@ -2,15 +2,15 @@ {% if task.status == "requested" %}

{{ translate('email.requested.title')}}

-

{{ translate('email.requested.paragraph1', link='' + task.config.flags.url + '') | safe}}

-

{{ translate('email.requested.paragraph2', link='' + base_url + '/#/request/' + task.id + '') | safe}}

-

{{ translate('email.requested.paragraph3')}}

+

{{ translate('email.requested.zimRequestRecorded', link='' + task.config.flags.url + '') | safe}}

+

{{ translate('email.requested.howToFollowProgress', link='' + base_url + '/#/request/' + task.id + '') | safe}}

+

{{ translate('email.requested.emailWillBeSent')}}

{% endif %} {% if task.status == "succeeded" %}

{{ translate('email.succeeded.title')}}

-

{{ translate('email.succeeded.paragraph1', link='' + task.config.flags.url + '') | safe}}

-

{{ translate('email.succeeded.paragraph2')}}

+

{{ translate('email.succeeded.zimRequestCompleted', link='' + task.config.flags.url + '') | safe}}

+

{{ translate('email.succeeded.hereItIs')}}

{% if task.files %}{% endif %} @@ -20,8 +20,8 @@

{{ translate('email.succeeded.title')}}

{% if task.status in ("failed", "canceled") %}

{{ translate('email.failed.title')}}

-

{{ translate('email.failed.paragraph1')}}

-

{{ translate('email.failed.paragraph2', configUrlLink='' + task.config.flags.url + '', retryLink='' + translate('email.failed.retryLinkContent') + '') | safe}}

-

{{ translate('email.failed.paragraph3', taskLink='' + base_url + '/#/request/' + task.id + '') | safe}}

{% endif %} +

{{ translate('email.failed.weAreSorry')}}

+

{{ translate('email.failed.checkAndRetry', configUrlLink='' + task.config.flags.url + '', retryLink='' + translate('email.failed.retryLinkContent') + '') | safe}}

+

{{ translate('email.failed.howToCheckSettings', taskLink='' + base_url + '/#/request/' + task.id + '') | safe}}

{% endif %} diff --git a/api/src/zimitfrontend/utils.py b/api/src/zimitfrontend/utils.py index 989b6a7..7228847 100644 --- a/api/src/zimitfrontend/utils.py +++ b/api/src/zimitfrontend/utils.py @@ -19,22 +19,21 @@ ) jinja_env.filters["short_id"] = lambda value: str(value)[:5] jinja_env.filters["format_size"] = lambda value: humanfriendly.format_size( - value, binary=True + value, binary=True # pyright: ignore[reportArgumentType] ) jinja_env.filters["format_timespan"] = lambda value: humanfriendly.format_timespan( - value + value # pyright: ignore[reportArgumentType] ) -jinja_env.globals["translate"] = i18n.t +jinja_env.globals["translate"] = i18n.t # pyright: ignore def send_email_via_mailgun( to: Sequence[str], subject: str, contents: str, - cc: Sequence | None = None, - bcc: Sequence | None = None, - attachments: Sequence | None = None, -): + cc: Sequence[str] | None = None, + bcc: Sequence[str] | None = None, +) -> str | None: """Send email via mailgun and return task id""" if not ApiConfiguration.mailgun_api_url or not ApiConfiguration.mailgun_api_key: logger.warning("Email not sent, Mailgun is not properly configured") @@ -51,14 +50,6 @@ def send_email_via_mailgun( "cc": cc, # can be a list "bcc": bcc, # can be a list }, - files=( - [ - ("attachment", (fpath.name, open(fpath, "rb").read())) - for fpath in attachments - ] - if attachments - else [] - ), timeout=ApiConfiguration.mailgun_requests_timeout, ) resp.raise_for_status() diff --git a/api/src/zimitfrontend/zimfarm.py b/api/src/zimitfrontend/zimfarm.py index 8801f8a..9571826 100644 --- a/api/src/zimitfrontend/zimfarm.py +++ b/api/src/zimitfrontend/zimfarm.py @@ -1,8 +1,9 @@ import datetime import json import logging +from collections.abc import Callable from http import HTTPStatus -from typing import Any +from typing import Any, ParamSpec, TypeVar import requests @@ -27,20 +28,20 @@ class ZimfarmAPIError(Exception): pass -def get_url(path): +def get_url(path: str) -> str: return "/".join( [ApiConfiguration.zimfarm_api_url, path[1:] if path[0] == "/" else path] ) -def get_token_headers(token): +def get_token_headers(token: str) -> dict[str, str]: return { "Authorization": f"Token {token}", "Content-type": "application/json", } -def get_token(username, password): +def get_token(username: str, password: str) -> tuple[str, str]: req = requests.post( url=get_url("/auth/authorize"), headers={ @@ -54,7 +55,7 @@ def get_token(username, password): return req.json().get("access_token"), req.json().get("refresh_token") -def authenticate(*, force=False): +def authenticate(*, force: bool = False) -> None: if ( not force and TokenData.ACCESS_TOKEN is not None @@ -85,8 +86,15 @@ def authenticate(*, force=False): ) + datetime.timedelta(days=29) -def auth_required(func): - def wrapper(*args, **kwargs): +# Generic type variable for the return type of the wrapped function +R = TypeVar("R") + +# ParamSpec for capturing the signature of the wrapped function +P = ParamSpec("P") + + +def auth_required(func: Callable[P, R]) -> Callable[P, R]: + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: authenticate() return func(*args, **kwargs) @@ -95,8 +103,14 @@ def wrapper(*args, **kwargs): @auth_required def query_api( - method: str, path, payload=None, params=None + method: str, path: str, payload: Any | None = None, params: Any | None = None ) -> tuple[bool, HTTPStatus, Any]: + if not TokenData.ACCESS_TOKEN: + return ( + False, + HTTPStatus.INTERNAL_SERVER_ERROR, + "Authentication on Zimfarm failed", + ) try: req = requests.request( method.lower(), @@ -111,7 +125,7 @@ def query_api( return (False, HTTPStatus.REQUEST_TIMEOUT, f"ConnectionError -- {exc}") try: - resp = req.json() if req.text else {} + resp: Any = req.json() if req.text else {} except json.JSONDecodeError: return ( False, diff --git a/api/tests/unit/routes/test_utils.py b/api/tests/unit/routes/test_utils.py index 4819aa7..4df3a78 100644 --- a/api/tests/unit/routes/test_utils.py +++ b/api/tests/unit/routes/test_utils.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from zimitfrontend.constants import ApiConfiguration @@ -6,7 +8,7 @@ FAILED, SUCCESS, convert_hook_to_mail, - convert_zimfarm_task_to_info, + get_task_info, ) @@ -93,8 +95,8 @@ ), ], ) -def test_convert_zimfarm_task_to_info(task, expected): - assert convert_zimfarm_task_to_info(task) == expected +def test_convert_zimfarm_task_to_info(task: Any, expected: TaskInfo): + assert get_task_info(task) == expected DEFAULT_HOOK_TASK = ZimfarmTask.model_validate( @@ -299,7 +301,14 @@ def test_convert_zimfarm_task_to_info(task, expected): ), ], ) -def test_convert_hook_to_mail(token, target, lang, task, task_status, expected): +def test_convert_hook_to_mail( + token: str | None, + target: str | None, + lang: str, + task: ZimfarmTask | None, + task_status: str, + expected: MailToSend, +): if task: task.status = task_status result = convert_hook_to_mail( diff --git a/dev/README.md b/dev/README.md index 82b2d2d..02017b9 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,23 +1,30 @@ This is a docker-compose configuration to be used **only** for development purpose. There is almost zero security in the stack configuration. -It is composed of the Zimit frontend and API (of course), but also a local Zimfarm DB, +It is composed of the Zimit frontend UI and API (of course), but also a local Zimfarm DB, API and UI, so that you can test the whole integration locally. -Zimit UI and API are not deployed as they would be in production to allow hot reload of +Zimit frontend UI has two containers, one identical to production and one allowing hot reload +of local developments. + +Zimit frontend API has one container, slightly modified to allow hot reload of most modifications done to the source code. Zimfarm UI, API and DB are deployed with official production Docker images. ## List of containers -### zimit_ui +### zimit_ui_prod + +This container is Zimit frontend UI as served in production (already compiled as a static website, so not possible to live-edit) -This container is Zimit frontend web server (UI only) +### zimit_ui_dev + +This container is Zimit frontend UI served in development mode (possible to live-edit) ### zimit_api -This container is Zimit API server (API only) +This container is Zimit frontend API server (only slightly modified to enable live reload of edits) ## zimfarm_db @@ -51,16 +58,19 @@ If you have requested a task via Zimit UI and want to simulate a worker starting ## Restart the backend Should the API process fail, you might restart it with: + ```sh -docker restart zimit-zimit_ui-1 +docker restart zimit-zimit_api-1 ``` ## Browse the web UIs You might open following URLs in your favorite browser: -- [Zimit UI](http://localhost:8001) -- [Zimfarm API](http://localhost:8002) +- [Zimit UI Prod](http://localhost:8000) +- [Zimit UI Dev](http://localhost:8001) +- [Zimit API](http://localhost:8002) - [Zimfarm UI](http://localhost:8003) +- [Zimfarm API](http://localhost:8004) -You can login into Zimfarm UI with username `admin` and password `admin`. \ No newline at end of file +You can login into Zimfarm UI with username `admin` and password `admin`. diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index e95bc45..627bcfc 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -30,7 +30,9 @@ services: depends_on: - zimfarm_api zimit_api: - build: .. + build: + dockerfile: Dockerfile-api + context: .. volumes: - ../api/src/zimitfrontend:/usr/local/lib/python3.12/site-packages/zimitfrontend - ../locales:/locales @@ -54,7 +56,7 @@ services: HOOK_TOKEN: a_very_secret_token depends_on: - zimfarm_api - zimit_ui: + zimit_ui_dev: build: dockerfile: dev/zimit_ui_dev/Dockerfile context: .. @@ -64,8 +66,14 @@ services: - ../dev/zimit_ui_dev/config.json:/app/public/config.json ports: - 127.0.0.1:8001:80 - environment: - ZIMIT_API_URL: http://localhost:8002 + depends_on: + - zimit_api + zimit_ui_prod: + build: + dockerfile: Dockerfile-ui + context: .. + ports: + - 127.0.0.1:8000:80 depends_on: - zimit_api diff --git a/locales/en.json b/locales/en.json index e48aae2..66074a6 100644 --- a/locales/en.json +++ b/locales/en.json @@ -20,7 +20,7 @@ } }, "footer": { - "paragraph": "Powered by {0} and {1}, thanks to a {2} Award ❤️", + "poweredByThankTo": "Powered by {0} and {1}, thanks to a {2} Award ❤️", "link0": "Kiwix", "link1": "Webrecorder", "link2": "Mozilla Open-Source Support" @@ -35,7 +35,8 @@ "fetchingDefinition": "Loading offliner definition…", "errorFetchingDefinition": "Error fetching offliner definition", "creatingRequest": "Creating request…", - "errorCreatingRequest": "Error creating request" + "errorCreatingRequest": "Error creating request", + "offlinerNotFound": "Zimit offliner not found, we probably experience a serious issue on our infrastructure." }, "notFound": { "heading": "Not Found", @@ -69,25 +70,25 @@ "requested": { "subject": "Youzim.it task %{taskId} requested", "title": "ZIM requested!", - "paragraph1": "Your ZIM request of %{link} has been recorded.", - "paragraph2": "You can follow the status of this request at %{link}.", - "paragraph3": "We'll send you another email once your ZIM file is ready to download." + "zimRequestRecorded": "Your ZIM request of %{link} has been recorded.", + "howToFollowProgress": "You can follow the status of this request at %{link}.", + "emailWillBeSent": "We'll send you another email once your ZIM file is ready to download." }, "succeeded": { "subject": "Youzim.it task %{taskId} succeeded", "title": "ZIM is ready!", - "paragraph1": "Your ZIM request of %{link} has completed.", - "paragraph2": "Here it is:", + "zimRequestCompleted": "Your ZIM request of %{link} has completed.", + "hereItIs": "Here it is:", "incomplete": "ZIM is unfortunately incomplete because you have reached the limits (%{sizeLimit} or %{timeLimit}) allowed for free crawling. %{contactUsLink} to help us purchase additional server space for you.", "contactUsLinkContent": "Contact us" }, "failed": { "subject": "Youzim.it task %{taskId} failed", "title": "Your ZIM request has failed!", - "paragraph1": "We are really sorry.", - "paragraph2": "This might be due to an error with the URL you entered (%{configUrlLink}) or additional settings missing / failing. Please double check and %{retryLink}.", + "weAreSorry": "We are really sorry.", + "checkAndRetry": "This might be due to an error with the URL you entered (%{configUrlLink}) or additional settings missing / failing. Please double check and %{retryLink}.", "retryLinkContent": "try again", - "paragraph3": "You can check the settings you used at %{taskLink}." + "howToCheckSettings": "You can check the settings you used at %{taskLink}." } } } diff --git a/locales/fr.json b/locales/fr.json index 5634fc4..53c4ad1 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -68,25 +68,25 @@ "requested": { "subject": "Tâche Youzim.it %{taskId} demandée", "title": "ZIM demandé !", - "paragraph1": "Votre demande de ZIM de %{link} a été enregistrée.", - "paragraph2": "Vous pouvez suivre l'état de cette demande à %{link}.", - "paragraph3": "Nous vous enverrons un autre e-mail une fois que votre fichier ZIM sera prêt à être téléchargé." + "zimRequestRecorded": "Votre demande de ZIM de %{link} a été enregistrée.", + "howToFollowProgress": "Vous pouvez suivre l'état de cette demande à %{link}.", + "emailWillBeSent": "Nous vous enverrons un autre e-mail une fois que votre fichier ZIM sera prêt à être téléchargé." }, "succeeded": { "subject": "Tâche Youzim.it %{taskId} réussie", "title": "ZIM est prêt !", - "paragraph1": "Votre demande de ZIM de %{link} est terminée.", - "paragraph2": "Le voici :", + "zimRequestCompleted": "Votre demande de ZIM de %{link} est terminée.", + "hereItIs": "Le voici :", "incomplete": "ZIM est malheureusement incomplet car vous avez atteint les limites (%{sizeLimit} ou %{timeLimit}) autorisées pour le crawling gratuit. %{contactUsLink} pour nous aider à acheter de l'espace serveur supplémentaire pour vous.", "contactUsLinkContent": "Contactez-nous" }, "failed": { "subject": "Tâche Youzim.it %{taskId} échouée", "title": "Votre demande de ZIM a échoué !", - "paragraph1": "Nous sommes vraiment désolés.", - "paragraph2": "Cela pourrait être dû à une erreur avec l'URL que vous avez entrée (%{configUrlLink}) ou à des paramètres supplémentaires manquants / défaillants. Veuillez vérifier et %{retryLink}.", + "weAreSorry": "Nous sommes vraiment désolés.", + "checkAndRetry": "Cela pourrait être dû à une erreur avec l'URL que vous avez entrée (%{configUrlLink}) ou à des paramètres supplémentaires manquants / défaillants. Veuillez vérifier et %{retryLink}.", "retryLinkContent": "réessayer", - "paragraph3": "Vous pouvez vérifier les paramètres que vous avez utilisés à %{taskLink}." + "howToCheckSettings": "Vous pouvez vérifier les paramètres que vous avez utilisés à %{taskLink}." } } } diff --git a/locales/qqq.json b/locales/qqq.json index d0daaaa..0b08f15 100644 --- a/locales/qqq.json +++ b/locales/qqq.json @@ -20,7 +20,7 @@ } }, "footer": { - "paragraph": "This is the main page footer message, with placeholders for links.", + "poweredByThankTo": "This is the main page footer message, with placeholders for links.", "link0": "This is the word(s) inside the first link of footer paragraph", "link1": "This is the word(s) inside the second link of footer paragraph", "link2": "This is the word(s) inside the third link of footer paragraph" @@ -35,7 +35,8 @@ "fetchingDefinition": "This is the message while fetching the task definition.", "errorFetchingDefinition": "This is the message when fetching the task definition failed.", "creatingRequest": "This is the message while creating a Zimfarm request.", - "errorCreatingRequest": "This is the message when creating a Zimfarm request failed." + "errorCreatingRequest": "This is the message when creating a Zimfarm request failed.", + "offlinerNotFound": "This is the message when we failed to load offliner definition through API call." }, "notFound": { "heading": "This is the heading displayed when URL is not found/handled.", @@ -69,25 +70,25 @@ "requested": { "subject": "This is the email subject when task has been requested", "title": "This is the email title (in body) when task has been requested", - "paragraph1": "This is the email paragraph #1 when task has been requested", - "paragraph2": "This is the email paragraph #2 when task has been requested", - "paragraph3": "This is the email paragraph #3 when task has been requested" + "zimRequestRecorded": "This explains that task has been requested", + "howToFollowProgress": "This explains how to follow task progress", + "emailWillBeSent": "This explains that an email will be sent when task is done" }, "succeeded": { "subject": "This is the email subject when task has succeeded", - "title": "This is the email title (in body) when task has been requested", - "paragraph1": "This is the email paragraph #1 when task has been requested", - "paragraph2": "This is the email paragraph #2 when task has been requested", - "incomplete": "This is an optional email paragraph when task has been requested but is incomplete", - "contactUsLinkContent": "This is the content of a link to contact us, used in a paragraph when task is incomplete" + "title": "This is the email title (in body) when task has succeeded", + "zimRequestCompleted": "This explains that request has completed", + "hereItIs": "This is the text before the download link(s)", + "incomplete": "This is an optional email paragraph when task has succeeded but is incomplete", + "contactUsLinkContent": "This is the content of a link to contact us, used in 'incomplete' paragraph when task is incomplete" }, "failed": { - "subject": "This is the email subject when task has succeeded", - "title": "This is the email title (in body) when task has been requested", - "paragraph1": "This is the email paragraph #1 when task has been requested", - "paragraph2": "This is the email paragraph #2 when task has been requested", + "subject": "This is the email subject when task has failed", + "title": "This is the email title (in body) when task has failed", + "weAreSorry": "This says that we are sorry for the failed task", + "checkAndRetry": "This explains that settings should be checked and retried", "retryLinkContent": "This is the content of a link to try request again", - "paragraph3": "This is the email paragraph #3 when task has been requested" + "howToCheckSettings": "This explains how to double-check the settings" } } } diff --git a/ui/src/App.vue b/ui/src/App.vue index 07105e8..2888de6 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -55,12 +55,12 @@ watch( > - + {{ t('footer.link0') }} {{ t('footer.link1') }} {{ t('footer.link2') }} - + {{ mainStore.snackbarContent }} diff --git a/ui/src/stores/main.ts b/ui/src/stores/main.ts index a4b8952..ff79d6f 100644 --- a/ui/src/stores/main.ts +++ b/ui/src/stores/main.ts @@ -9,6 +9,7 @@ export type RootState = { loading: boolean loadingText: string offlinerDefinition: OfflinerDefinition | undefined + offlinerNotFound: boolean formValues: NameValue[] taskId: string taskData: TaskData | undefined @@ -70,6 +71,7 @@ export const useMainStore = defineStore('main', { loading: false, loadingText: '', offlinerDefinition: undefined, + offlinerNotFound: false, formValues: [] as NameValue[], taskId: '', taskData: undefined, @@ -142,8 +144,10 @@ export const useMainStore = defineStore('main', { (flag) => this.config.new_request_advanced_flags.indexOf(flag.key) > -1 ) this.offlinerDefinition = offlinerDefinition + this.offlinerNotFound = false } catch (error) { this.handleError(this.t('newRequest.errorFetchingDefinition'), error) + this.offlinerNotFound = true } finally { this.setLoading({ loading: false }) } diff --git a/ui/src/style.css b/ui/src/style.css index 0129008..4b12ae5 100644 --- a/ui/src/style.css +++ b/ui/src/style.css @@ -59,3 +59,7 @@ h1 { font-weight: 500; font-size: 2.5rem; } + +.red { + color: red; +} diff --git a/ui/src/views/NewRequest.vue b/ui/src/views/NewRequest.vue index a937958..eb09443 100644 --- a/ui/src/views/NewRequest.vue +++ b/ui/src/views/NewRequest.vue @@ -21,6 +21,9 @@ onMounted(() => {
{{ mainStore.loadingText }}
+
+ {{ $t('newRequest.offlinerNotFound') }} +
diff --git a/ui/src/views/RequestStatus.vue b/ui/src/views/RequestStatus.vue index be76020..1a17f58 100644 --- a/ui/src/views/RequestStatus.vue +++ b/ui/src/views/RequestStatus.vue @@ -166,8 +166,4 @@ watch( th { width: 25%; } - -.red { - color: red; -}