diff --git a/Dockerfile b/Dockerfile index 5041a55..f90a328 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,4 @@ COPY --from=base-poetry /src/requirements.txt ./requirements.txt RUN pip install -r requirements.txt COPY ./app app EXPOSE 5000 -CMD uvicorn app:app --host 0.0.0.0 --port 5000 --use-colors +CMD uvicorn app.main:app --host 0.0.0.0 --port 5000 --use-colors diff --git a/app/__init__.py b/app/__init__.py index 7689974..6265d82 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,34 +1,6 @@ # -*- coding: utf-8 -*- from logging.config import dictConfig -from fastapi import FastAPI, HTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse - -from . import views from .logging_config import LogConfig dictConfig(LogConfig().dict()) - -app = FastAPI() - - -@app.exception_handler(400) -@app.exception_handler(401) -@app.exception_handler(404) -async def exception_handler(request: Request, exc: HTTPException) -> JSONResponse: - return JSONResponse( - status_code=exc.status_code, - content=dict( - status="error", - error=exc.detail, - ), - ) - - -@app.get("/ping", response_model=str) -def ping() -> str: - return "pong" - - -app.include_router(views.router_api) diff --git a/app/logging_config.py b/app/logging_config.py index 7ee8c6c..4ff0a69 100644 --- a/app/logging_config.py +++ b/app/logging_config.py @@ -5,7 +5,7 @@ class LogConfig(BaseModel): """Logging configuration""" LOGGER_NAME: str = "image-storage" - LOG_FORMAT: str = "%(levelprefix)s | %(asctime)s | %(name)s | %(message)s" + LOG_FORMAT: str = "%(levelprefix)s | %(asctime)s.%(msecs)03d | %(name)s | %(message)s" LOG_LEVEL: str = "DEBUG" # Logging config @@ -16,6 +16,7 @@ class LogConfig(BaseModel): "()": "uvicorn.logging.DefaultFormatter", "fmt": LOG_FORMAT, "datefmt": "%Y-%m-%d %H:%M:%S", + "use_colors": True, }, } handlers = { @@ -24,13 +25,13 @@ class LogConfig(BaseModel): "class": "logging.StreamHandler", "stream": "ext://sys.stderr", }, - "image-storage": { + LOGGER_NAME: { "formatter": "default", "class": "logging.StreamHandler", "stream": "ext://sys.stdout", }, } loggers = { - "image-storage": {"handlers": ["image-storage"], "level": LOG_LEVEL}, - "update_db": {"handlers": ["image-storage"], "level": LOG_LEVEL}, + LOGGER_NAME: {"handlers": [LOGGER_NAME], "level": LOG_LEVEL}, + "uvicorn": {"handlers": [LOGGER_NAME], "level": "INFO"}, } diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..071db44 --- /dev/null +++ b/app/main.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +import asyncio +import logging +import os +from contextlib import asynccontextmanager +from typing import AsyncIterator + +from fastapi import FastAPI, HTTPException +from starlette.concurrency import run_in_threadpool +from starlette.requests import Request +from starlette.responses import JSONResponse + +from . import views +from .settings import settings +from .storage_manager import S3StorageManager, StorageManager + +logger = logging.getLogger("image-storage") + + +async def put_images_to_s3() -> None: + file_sm = StorageManager() + s3_sm = S3StorageManager() + all_uuids = await run_in_threadpool(s3_sm.list_uuids) + for file1 in os.listdir(settings.upload_folder): + await asyncio.sleep(0) + path1 = os.path.join(settings.upload_folder, file1) + for file2 in os.listdir(path1): + await asyncio.sleep(0) + uuid = [file1, file2] + if uuid in all_uuids: + logger.debug(f"Already exists {uuid}") + continue + image = file_sm.get_image(uuid) + s3_sm.save_image(uuid, image.data) + logger.debug(f"Moved {uuid}") + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator: + # noinspection PyAsyncCall + # asyncio.create_task(put_images_to_s3()) + yield + + +app = FastAPI(lifespan=lifespan) + + +@app.exception_handler(400) +@app.exception_handler(401) +@app.exception_handler(404) +async def exception_handler(request: Request, exc: HTTPException) -> JSONResponse: + return JSONResponse( + status_code=exc.status_code, + content=dict( + status="error", + error=exc.detail, + ), + ) + + +@app.get("/ping", response_model=str) +def ping() -> str: + return "pong" + + +app.include_router(views.router_api) diff --git a/app/settings.py b/app/settings.py index 183bb39..563954b 100644 --- a/app/settings.py +++ b/app/settings.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from typing import Optional + from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict @@ -8,11 +10,20 @@ class ClientInfo(BaseModel): api_key: str +class S3Config(BaseModel): + url: Optional[str] = None + access_key: str = "" + secret_key: str = "" + bucket: str = "" + + class Settings(BaseSettings): model_config = SettingsConfigDict(env_nested_delimiter="__") upload_folder: str = "" clients_info: dict[str, ClientInfo] = {} + s3: S3Config = S3Config() + settings = Settings() diff --git a/app/storage_manager.py b/app/storage_manager.py index ade82bb..9643f0f 100644 --- a/app/storage_manager.py +++ b/app/storage_manager.py @@ -1,14 +1,36 @@ # -*- coding: utf-8 -*- -import json +import io +import logging import os +from dataclasses import dataclass +from functools import cache + +import boto3 +from botocore.exceptions import ClientError +from mypy_boto3_s3.service_resource import Object +from PIL import Image as PILImage from .settings import settings +logger = logging.getLogger("image-storage") + class StorageManagerException(Exception): pass +@dataclass +class Image: + data: bytes + mimetype: str + + +def get_content_type_for_data(data: bytes) -> str: + pil_image = PILImage.open(io.BytesIO(data)) + assert pil_image.format is not None + return PILImage.MIME[pil_image.format] + + class StorageManager(object): def path_for_uuid(self, uuid: list[str]) -> str: return os.path.join(settings.upload_folder, *uuid) @@ -16,31 +38,75 @@ def path_for_uuid(self, uuid: list[str]) -> str: def uuid_exists(self, uuid: list[str]) -> bool: return os.path.exists(self.path_for_uuid(uuid)) - def save_image(self, uuid: list[str], file_content: bytes, data: str | None = None) -> None: + def save_image(self, uuid: list[str], file_content: bytes) -> None: folder = self.path_for_uuid(uuid) os.makedirs(folder) with open(os.path.join(folder, "file"), "wb") as f: f.write(file_content) - if data: - with open(os.path.join(folder, "data"), "w") as f: - f.write(data) - - def _read_data(self, uuid: list[str]) -> dict[str, str]: - path = os.path.join(self.path_for_uuid(uuid), "data") - if not os.path.exists(path): - return {} - with open(path, "r") as f: - return json.loads(f.read()) - - def _read_file(self, uuid: list[str]) -> bytes: - with open(os.path.join(self.path_for_uuid(uuid), "file"), "rb") as f: - return f.read() - - def get_file(self, uuid: list[str]) -> tuple[bytes, dict]: + + def get_image(self, uuid: list[str]) -> Image: if not self.uuid_exists(uuid): raise StorageManagerException("Not Found") - return self._read_file(uuid), self._read_data(uuid) + filename = os.path.join(self.path_for_uuid(uuid), "file") + with open(filename, "rb") as f: + data = f.read() + pil_image = PILImage.open(io.BytesIO(data)) + assert pil_image.format is not None + return Image( + data=data, + mimetype=PILImage.MIME[pil_image.format], + ) + + +class S3StorageManager(object): + def __init__(self) -> None: + s3_resource = boto3.resource( + "s3", + endpoint_url=settings.s3.url, + aws_access_key_id=settings.s3.access_key, + aws_secret_access_key=settings.s3.secret_key, + aws_session_token=None, + ) + self.bucket = s3_resource.Bucket(settings.s3.bucket) + + def object_for_uuid(self, uuid: list[str]) -> Object: + return self.bucket.Object("/".join(uuid)) + + def save_image(self, uuid: list[str], file_content: bytes) -> None: + self.object_for_uuid(uuid).put( + Body=file_content, + ContentType=get_content_type_for_data(file_content), + ) + + def list_uuids(self) -> list[list[str]]: + return [object_summary.key.split("/") for object_summary in self.bucket.objects.all()] + + def uuid_exists(self, uuid: list[str]) -> bool: + try: + self.object_for_uuid(uuid).load() + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + return False + raise + + def get_image(self, uuid: list[str]) -> Image: + try: + logger.debug("logger - get image start") + data = self.object_for_uuid(uuid).get()["Body"].read() + mimetype = get_content_type_for_data(data) + logger.debug("logger - get image end") + return Image( + data=data, + mimetype=mimetype, + ) + except ClientError as e: + if e.response["Error"]["Code"] == "NoSuchKey": + return StorageManager().get_image(uuid) + # raise StorageManagerException("Not Found") + raise -def get_storage_manager() -> StorageManager: - return StorageManager() +@cache +def get_storage_manager() -> S3StorageManager: + return S3StorageManager() diff --git a/app/views.py b/app/views.py index 876b05a..8e12b82 100644 --- a/app/views.py +++ b/app/views.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import base64 -import json import logging from fastapi import APIRouter, HTTPException @@ -22,7 +21,7 @@ class PostImageRequest(BaseModel): @router_api.post("/image/") -async def post_image( +def post_image( request: Request, params: PostImageRequest, ) -> dict[str, str]: @@ -50,11 +49,6 @@ async def post_image( get_storage_manager().save_image( uuid=[client.id, filename], file_content=base64.b64decode(params.base64), - data=json.dumps( - dict( - mimetype="image/jpeg", - ) - ), ) return dict( @@ -64,10 +58,10 @@ async def post_image( @router_api.get("/image/{client_id}/{uuid}") -async def get_image(client_id: str, uuid: str) -> Response: +def get_image(client_id: str, uuid: str) -> Response: try: - content, data = get_storage_manager().get_file([client_id, uuid]) - return Response(content=content, media_type=data.get("mimetype")) + image = get_storage_manager().get_image([client_id, uuid]) + return Response(content=image.data, media_type=image.mimetype) except StorageManagerException as e: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/docker-compose.yaml b/docker-compose.yaml index 1ce8ae0..c38fdae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,11 +4,40 @@ services: context: . target: prod restart: always + depends_on: + create_access_key: + condition: service_completed_successfully environment: UPLOAD_FOLDER: /uploads CLIENTS_INFO__0__api_key: api_key CLIENTS_INFO__0__id: client + S3__URL: http://minio:9000/ + S3__ACCESS_KEY: access-key + S3__SECRET_KEY: secret-key + S3__BUCKET: bucket ports: - 5000:5000 volumes: - ./uploads:/uploads + + minio: + image: docker.io/bitnami/minio + ports: + - 9001:9001 + environment: + MINIO_ROOT_USER: username + MINIO_ROOT_PASSWORD: password + MINIO_DEFAULT_BUCKETS: bucket + MINIO_SCHEME: http + BITNAMI_DEBUG: true + + create_access_key: + image: minio/mc + depends_on: + - minio + restart: on-failure + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add myminio http://minio:9000 username password; + mc admin user svcacct add myminio username --access-key access-key --secret-key secret-key || true; + " diff --git a/pyproject.toml b/pyproject.toml index a467837..bced9a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,9 @@ fastapi = "~0.111" uvicorn = "~0.30" base58 = "2.1.1" pydantic-settings = "~2.2.1" +pillow = "~10.3.0" +boto3 = "~1.34.118" +boto3-stubs = {version = "~1.34.118", extras = ["s3"]} [tool.poetry.group.test] optional = true @@ -18,6 +21,8 @@ optional = true pytest = "~8.2.0" pytest-cov = "^5.0.0" pytest-asyncio = "^0" +pytest-socket = "~0.7.0" +moto = {version = "^5.0.9", extras = ["s3"]} [tool.poetry.group.dev] optional = true @@ -39,6 +44,7 @@ addopts =""" -vv -p no:cacheprovider --strict-markers + --disable-socket --allow-unix-socket """ [tool.black] diff --git a/tests/conftest.py b/tests/conftest.py index 198e0c8..d6bf5aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,23 @@ # -*- coding: utf-8 -*- import tempfile +import boto3 import pytest +from moto import mock_aws @pytest.fixture() def upload_folder() -> str: return tempfile.mkdtemp() + + +@pytest.fixture(scope="session", autouse=True) +def mock_s3_bucket(): + with mock_aws(): + conn = boto3.resource("s3") + yield conn.create_bucket(Bucket="mybucket") + + +@pytest.fixture(autouse=True) +def clean_s3_bucket(mock_s3_bucket): + mock_s3_bucket.objects.delete() diff --git a/tests/test_storage_manager.py b/tests/test_storage_manager.py index 3d8ddcf..d9bc4e0 100644 --- a/tests/test_storage_manager.py +++ b/tests/test_storage_manager.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- +import io import os import pytest +from PIL import Image from app import storage_manager -from app.settings import Settings +from app.settings import S3Config, Settings class TestStorageManager: @@ -14,26 +16,22 @@ def _setup(self, upload_folder, monkeypatch): self.upload_folder = upload_folder self.manager = storage_manager.StorageManager() - def check_save(self, uuid, data=None): + self.image = Image.new(mode="RGB", size=(3, 3)) + img_stream = io.BytesIO() + self.image.save(img_stream, format="jpeg") + self.image_byte_array = img_stream.getvalue() + + def check_save(self, uuid): folder = os.path.join(self.upload_folder, uuid) file_path = os.path.join(folder, "file") - data_path = os.path.join(folder, "data") assert os.path.isdir(folder) assert os.path.isfile(file_path) with open(file_path, "rb") as file: - assert file.read() == b"abcdef" - if data is not None: - assert os.path.isfile(data_path) - with open(data_path, "r") as file: - assert file.read() == data + assert file.read() == self.image_byte_array def test_save_image_ok(self): - self.manager.save_image(["some_uuid"], b"abcdef", "some_data") - self.check_save("some_uuid", "some_data") - - def test_save_image_no_data_ok(self): - self.manager.save_image(["some_uuid"], b"abcdef") + self.manager.save_image(["some_uuid"], self.image_byte_array) self.check_save("some_uuid") @pytest.mark.parametrize("exists", [True, False]) @@ -42,18 +40,55 @@ def test_exists(self, exists): os.makedirs(os.path.join(self.upload_folder, "some_uuid")) assert self.manager.uuid_exists(["some_uuid"]) == exists - def test_read_data(self): - os.makedirs(os.path.join(self.upload_folder, "some_uuid")) - with open(os.path.join(self.upload_folder, "some_uuid", "data"), "w") as f: - f.write('{"foo": "bar"}') - assert self.manager._read_data(["some_uuid"]) == {"foo": "bar"} - - def test_read_file(self): + def test_get_image(self): os.makedirs(os.path.join(self.upload_folder, "some_uuid")) with open(os.path.join(self.upload_folder, "some_uuid", "file"), "wb") as f: - f.write(b"READ DATA") - assert self.manager._read_file(["some_uuid"]) == b"READ DATA" + f.write(self.image_byte_array) + result_image = self.manager.get_image(["some_uuid"]) + assert result_image.data == self.image_byte_array + assert result_image.mimetype == "image/jpeg" def test_path_for_uuid_with_sep(self): path = self.manager.path_for_uuid(["foo", "bar"]) assert path == os.path.join(self.upload_folder, "foo", "bar") + + +class TestS3StorageManager: + @pytest.fixture(autouse=True) + def _setup(self, mock_s3_bucket, monkeypatch): + monkeypatch.setattr(storage_manager, "settings", Settings(s3=S3Config(bucket=mock_s3_bucket.name))) + self.bucket = mock_s3_bucket + self.manager = storage_manager.S3StorageManager() + + self.image = Image.new(mode="RGB", size=(3, 3)) + img_stream = io.BytesIO() + self.image.save(img_stream, format="jpeg") + self.image_byte_array = img_stream.getvalue() + + def test_save_image_ok(self): + self.manager.save_image(["client", "some_uuid"], self.image_byte_array) + created_object = self.bucket.Object("client/some_uuid").get() + assert created_object["Body"].read() == self.image_byte_array + assert created_object["ContentType"] == "image/jpeg" + + def test_get_image(self): + self.bucket.Object("some_uuid").put(Body=self.image_byte_array) + result_image = self.manager.get_image(["some_uuid"]) + assert result_image.data == self.image_byte_array + assert result_image.mimetype == "image/jpeg" + + def test_uuid_exists(self): + self.bucket.Object("some_uuid").put(Body=self.image_byte_array) + assert self.manager.uuid_exists(["some_uuid"]) is True + assert self.manager.uuid_exists(["some_other_uuid"]) is False + + def test_list_uuids(self): + assert self.manager.list_uuids() == [] + self.bucket.Object("some_uuid").put(Body=self.image_byte_array) + assert self.manager.list_uuids() == [["some_uuid"]] + self.bucket.Object("some/other/uuid").put(Body=self.image_byte_array) + assert sorted(self.manager.list_uuids()) == [["some", "other", "uuid"], ["some_uuid"]] + + def test_get_image_unknown_key(self): + with pytest.raises(storage_manager.StorageManagerException): + self.manager.get_image(["some_uuid"]) diff --git a/tests/views/conftest.py b/tests/views/conftest.py index b272e34..d91037f 100644 --- a/tests/views/conftest.py +++ b/tests/views/conftest.py @@ -2,16 +2,20 @@ import pytest_asyncio from httpx import ASGITransport, AsyncClient -from app import app, logic, storage_manager -from app.settings import ClientInfo, Settings +from app import logic, storage_manager +from app.main import app +from app.settings import ClientInfo, S3Config, Settings @pytest.fixture(autouse=True) -def mock_settings(monkeypatch, upload_folder): +def mock_settings(monkeypatch, upload_folder, mock_s3_bucket): # TODO: dynamically find all modules which use settings settings = Settings( clients_info={"0": ClientInfo(id="test_client", api_key="TEST_API_KEY")}, upload_folder=upload_folder, + s3=S3Config( + bucket=mock_s3_bucket.name, + ), ) monkeypatch.setattr(logic, "settings", settings) monkeypatch.setattr(storage_manager, "settings", settings) diff --git a/tests/views/test_images.py b/tests/views/test_images.py index 11aa96d..a48f47e 100644 --- a/tests/views/test_images.py +++ b/tests/views/test_images.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- import base64 -import json -import os +import io import pytest -import pytest_asyncio +from PIL import Image @pytest.fixture() @@ -17,10 +16,15 @@ def mock_func(suggested_filename): @pytest.mark.asyncio class TestImageViewPost: - @pytest_asyncio.fixture(autouse=True) - async def _setup(self, client, generate_image_uuid_mock, upload_folder): + @pytest.fixture(autouse=True) + def _setup(self, client, generate_image_uuid_mock, mock_s3_bucket): self.client = client - self.upload_folder = upload_folder + self.bucket = mock_s3_bucket + + self.image = Image.new(mode="RGB", size=(3, 3)) + img_stream = io.BytesIO() + self.image.save(img_stream, format="jpeg") + self.image_byte_array = img_stream.getvalue() async def test_no_api_key(self): response = await self.client.post("/v1/image/", json={}) @@ -46,7 +50,7 @@ async def test_no_file(self): ) async def test_ok(self, filename): data = dict() - data["base64"] = base64.b64encode(b"abcdef").decode() + data["base64"] = base64.b64encode(self.image_byte_array).decode() if filename: data["file_name"] = filename expected_filename = (filename if filename else "aaa") + "_generated" @@ -60,31 +64,27 @@ async def test_ok(self, filename): "status": "ok", "uuid": "test_client/{}".format(expected_filename), } - assert os.path.isdir(os.path.join(self.upload_folder, "test_client", expected_filename)) - with open(os.path.join(self.upload_folder, "test_client", expected_filename, "file"), "rb") as file: - assert file.read() == b"abcdef" - with open(os.path.join(self.upload_folder, "test_client", expected_filename, "data"), "r") as file: - assert json.loads(file.read()) == dict( - mimetype="image/jpeg", - ) + assert self.bucket.Object(f"test_client/{expected_filename}").get()["Body"].read() == self.image_byte_array @pytest.mark.asyncio class TestImageViewGet: - @pytest_asyncio.fixture(autouse=True) - def _setup(self, client, upload_folder): + @pytest.fixture(autouse=True) + def _setup(self, client, mock_s3_bucket): self.client = client - os.makedirs(os.path.join(upload_folder, "some_client_id", "some_uuid")) - with open(os.path.join(upload_folder, "some_client_id", "some_uuid", "file"), "wb") as f: - f.write(b"abcdef") - with open(os.path.join(upload_folder, "some_client_id", "some_uuid", "data"), "w") as f: - f.write(json.dumps(dict(mimetype="image/jpeg"))) + self.bucket = mock_s3_bucket + + image = Image.new(mode="RGB", size=(3, 3)) + img_stream = io.BytesIO() + image.save(img_stream, format="jpeg") + self.image_byte_array = img_stream.getvalue() + self.bucket.Object("some_client_id/some_uuid").put(Body=self.image_byte_array) async def test_ok(self): response = await self.client.get("/v1/image/some_client_id/some_uuid") assert response.status_code == 200 assert response.headers["Content-Type"] == "image/jpeg" - assert response.content == b"abcdef" + assert response.content == self.image_byte_array async def test_no_client_id(self): response = await self.client.get("/v1/image/some_uuid")