s3 backend

Dockerfile

@@ -14,4 +14,4 @@ COPY --from=base-poetry /src/requirements.txt ./requirements.txt
 RUN pip install -r requirements.txt
 COPY ./app app
 EXPOSE 5000
-CMD uvicorn app:app --host 0.0.0.0 --port 5000 --use-colors
+CMD uvicorn app.main:app --host 0.0.0.0 --port 5000 --use-colors

app/__init__.py

@@ -1,34 +1,6 @@
 # -*- coding: utf-8 -*-
 from logging.config import dictConfig
 
-from fastapi import FastAPI, HTTPException
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-
-from . import views
 from .logging_config import LogConfig
 
 dictConfig(LogConfig().dict())
-
-app = FastAPI()
-
-
-@app.exception_handler(400)
-@app.exception_handler(401)
-@app.exception_handler(404)
-async def exception_handler(request: Request, exc: HTTPException) -> JSONResponse:
-    return JSONResponse(
-        status_code=exc.status_code,
-        content=dict(
-            status="error",
-            error=exc.detail,
-        ),
-    )
-
-
-@app.get("/ping", response_model=str)
-def ping() -> str:
-    return "pong"
-
-
-app.include_router(views.router_api)

app/logging_config.py

@@ -5,7 +5,7 @@ class LogConfig(BaseModel):
     """Logging configuration"""
 
     LOGGER_NAME: str = "image-storage"
-    LOG_FORMAT: str = "%(levelprefix)s | %(asctime)s | %(name)s | %(message)s"
+    LOG_FORMAT: str = "%(levelprefix)s | %(asctime)s.%(msecs)03d | %(name)s | %(message)s"
     LOG_LEVEL: str = "DEBUG"
 
     # Logging config
@@ -16,6 +16,7 @@ class LogConfig(BaseModel):
             "()": "uvicorn.logging.DefaultFormatter",
             "fmt": LOG_FORMAT,
             "datefmt": "%Y-%m-%d %H:%M:%S",
+            "use_colors": True,
         },
     }
     handlers = {
@@ -24,13 +25,13 @@ class LogConfig(BaseModel):
             "class": "logging.StreamHandler",
             "stream": "ext://sys.stderr",
         },
-        "image-storage": {
+        LOGGER_NAME: {
             "formatter": "default",
             "class": "logging.StreamHandler",
             "stream": "ext://sys.stdout",
         },
     }
     loggers = {
-        "image-storage": {"handlers": ["image-storage"], "level": LOG_LEVEL},
-        "update_db": {"handlers": ["image-storage"], "level": LOG_LEVEL},
+        LOGGER_NAME: {"handlers": [LOGGER_NAME], "level": LOG_LEVEL},
+        "uvicorn": {"handlers": [LOGGER_NAME], "level": "INFO"},
     }

app/main.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from fastapi import FastAPI, HTTPException
+from starlette.concurrency import run_in_threadpool
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
+from . import views
+from .settings import settings
+from .storage_manager import S3StorageManager, StorageManager
+
+logger = logging.getLogger("image-storage")
+
+
+async def put_images_to_s3() -> None:
+    file_sm = StorageManager()
+    s3_sm = S3StorageManager()
+    all_uuids = await run_in_threadpool(s3_sm.list_uuids)
+    for file1 in os.listdir(settings.upload_folder):
+        await asyncio.sleep(0)
+        path1 = os.path.join(settings.upload_folder, file1)
+        for file2 in os.listdir(path1):
+            await asyncio.sleep(0)
+            uuid = [file1, file2]
+            if uuid in all_uuids:
+                logger.debug(f"Already exists {uuid}")
+                continue
+            image = file_sm.get_image(uuid)
+            s3_sm.save_image(uuid, image.data)
+            logger.debug(f"Moved {uuid}")
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncIterator:
+    # noinspection PyAsyncCall
+    # asyncio.create_task(put_images_to_s3())
+    yield
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+@app.exception_handler(400)
+@app.exception_handler(401)
+@app.exception_handler(404)
+async def exception_handler(request: Request, exc: HTTPException) -> JSONResponse:
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=dict(
+            status="error",
+            error=exc.detail,
+        ),
+    )
+
+
+@app.get("/ping", response_model=str)
+def ping() -> str:
+    return "pong"
+
+
+app.include_router(views.router_api)

app/settings.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from typing import Optional
+
 from pydantic import BaseModel
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -8,11 +10,20 @@ class ClientInfo(BaseModel):
     api_key: str
 
 
+class S3Config(BaseModel):
+    url: Optional[str] = None
+    access_key: str = ""
+    secret_key: str = ""
+    bucket: str = ""
+
+
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_nested_delimiter="__")
 
     upload_folder: str = ""
     clients_info: dict[str, ClientInfo] = {}
 
+    s3: S3Config = S3Config()
+
 
 settings = Settings()

app/storage_manager.py

@@ -1,46 +1,112 @@
 # -*- coding: utf-8 -*-
-import json
+import io
+import logging
 import os
+from dataclasses import dataclass
+from functools import cache
+
+import boto3
+from botocore.exceptions import ClientError
+from mypy_boto3_s3.service_resource import Object
+from PIL import Image as PILImage
 
 from .settings import settings
 
+logger = logging.getLogger("image-storage")
+
 
 class StorageManagerException(Exception):
     pass
 
 
+@dataclass
+class Image:
+    data: bytes
+    mimetype: str
+
+
+def get_content_type_for_data(data: bytes) -> str:
+    pil_image = PILImage.open(io.BytesIO(data))
+    assert pil_image.format is not None
+    return PILImage.MIME[pil_image.format]
+
+
 class StorageManager(object):
     def path_for_uuid(self, uuid: list[str]) -> str:
         return os.path.join(settings.upload_folder, *uuid)
 
     def uuid_exists(self, uuid: list[str]) -> bool:
         return os.path.exists(self.path_for_uuid(uuid))
 
-    def save_image(self, uuid: list[str], file_content: bytes, data: str | None = None) -> None:
+    def save_image(self, uuid: list[str], file_content: bytes) -> None:
         folder = self.path_for_uuid(uuid)
         os.makedirs(folder)
         with open(os.path.join(folder, "file"), "wb") as f:
             f.write(file_content)
-        if data:
-            with open(os.path.join(folder, "data"), "w") as f:
-                f.write(data)
-
-    def _read_data(self, uuid: list[str]) -> dict[str, str]:
-        path = os.path.join(self.path_for_uuid(uuid), "data")
-        if not os.path.exists(path):
-            return {}
-        with open(path, "r") as f:
-            return json.loads(f.read())
-
-    def _read_file(self, uuid: list[str]) -> bytes:
-        with open(os.path.join(self.path_for_uuid(uuid), "file"), "rb") as f:
-            return f.read()
-
-    def get_file(self, uuid: list[str]) -> tuple[bytes, dict]:
+
+    def get_image(self, uuid: list[str]) -> Image:
         if not self.uuid_exists(uuid):
             raise StorageManagerException("Not Found")
-        return self._read_file(uuid), self._read_data(uuid)
+        filename = os.path.join(self.path_for_uuid(uuid), "file")
+        with open(filename, "rb") as f:
+            data = f.read()
+        pil_image = PILImage.open(io.BytesIO(data))
+        assert pil_image.format is not None
+        return Image(
+            data=data,
+            mimetype=PILImage.MIME[pil_image.format],
+        )
+
+
+class S3StorageManager(object):
+    def __init__(self) -> None:
+        s3_resource = boto3.resource(
+            "s3",
+            endpoint_url=settings.s3.url,
+            aws_access_key_id=settings.s3.access_key,
+            aws_secret_access_key=settings.s3.secret_key,
+            aws_session_token=None,
+        )
+        self.bucket = s3_resource.Bucket(settings.s3.bucket)
+
+    def object_for_uuid(self, uuid: list[str]) -> Object:
+        return self.bucket.Object("/".join(uuid))
+
+    def save_image(self, uuid: list[str], file_content: bytes) -> None:
+        self.object_for_uuid(uuid).put(
+            Body=file_content,
+            ContentType=get_content_type_for_data(file_content),
+        )
+
+    def list_uuids(self) -> list[list[str]]:
+        return [object_summary.key.split("/") for object_summary in self.bucket.objects.all()]
+
+    def uuid_exists(self, uuid: list[str]) -> bool:
+        try:
+            self.object_for_uuid(uuid).load()
+            return True
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "404":
+                return False
+            raise
+
+    def get_image(self, uuid: list[str]) -> Image:
+        try:
+            logger.debug("logger - get image start")
+            data = self.object_for_uuid(uuid).get()["Body"].read()
+            mimetype = get_content_type_for_data(data)
+            logger.debug("logger - get image end")
+            return Image(
+                data=data,
+                mimetype=mimetype,
+            )
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "NoSuchKey":
+                return StorageManager().get_image(uuid)
+                # raise StorageManagerException("Not Found")
+            raise
 
 
-def get_storage_manager() -> StorageManager:
-    return StorageManager()
+@cache
+def get_storage_manager() -> S3StorageManager:
+    return S3StorageManager()

app/views.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 import base64
-import json
 import logging
 
 from fastapi import APIRouter, HTTPException
@@ -22,7 +21,7 @@ class PostImageRequest(BaseModel):
 
 
 @router_api.post("/image/")
-async def post_image(
+def post_image(
     request: Request,
     params: PostImageRequest,
 ) -> dict[str, str]:
@@ -50,11 +49,6 @@ async def post_image(
     get_storage_manager().save_image(
         uuid=[client.id, filename],
         file_content=base64.b64decode(params.base64),
-        data=json.dumps(
-            dict(
-                mimetype="image/jpeg",
-            )
-        ),
     )
 
     return dict(
@@ -64,10 +58,10 @@ async def post_image(
 
 
 @router_api.get("/image/{client_id}/{uuid}")
-async def get_image(client_id: str, uuid: str) -> Response:
+def get_image(client_id: str, uuid: str) -> Response:
     try:
-        content, data = get_storage_manager().get_file([client_id, uuid])
-        return Response(content=content, media_type=data.get("mimetype"))
+        image = get_storage_manager().get_image([client_id, uuid])
+        return Response(content=image.data, media_type=image.mimetype)
     except StorageManagerException as e:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,

docker-compose.yaml

@@ -4,11 +4,40 @@ services:
       context: .
       target: prod
     restart: always
+    depends_on:
+      create_access_key:
+        condition: service_completed_successfully
     environment:
       UPLOAD_FOLDER: /uploads
       CLIENTS_INFO__0__api_key: api_key
       CLIENTS_INFO__0__id: client
+      S3__URL: http://minio:9000/
+      S3__ACCESS_KEY: access-key
+      S3__SECRET_KEY: secret-key
+      S3__BUCKET: bucket
     ports:
       - 5000:5000
     volumes:
       - ./uploads:/uploads
+
+  minio:
+    image: docker.io/bitnami/minio
+    ports:
+      - 9001:9001
+    environment:
+      MINIO_ROOT_USER: username
+      MINIO_ROOT_PASSWORD: password
+      MINIO_DEFAULT_BUCKETS: bucket
+      MINIO_SCHEME: http
+      BITNAMI_DEBUG: true
+
+  create_access_key:
+    image: minio/mc
+    depends_on:
+      - minio
+    restart: on-failure
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host add myminio http://minio:9000 username password;
+      mc admin user svcacct add myminio username --access-key access-key --secret-key secret-key || true;
+      "