Merge pull request #16 from bento-platform/modernize

Dockerfiles and switch to async (Quart)

.github/workflows/build.yml

@@ -0,0 +1,32 @@
+name: Build and push bento_drop_box_service
+on:
+  release:
+    types: [ published ]
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  build-push:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Run Bento build action
+        uses: bento-platform/bento_build_action@v0.6
+        with:
+          registry: ghcr.io
+          registry-username: ${{ github.actor }}
+          registry-password: ${{ secrets.GITHUB_TOKEN }}
+          image-name: ghcr.io/bento-platform/bento_drop_box_service
+          development-dockerfile: dev.Dockerfile
+          dockerfile: Dockerfile

.github/workflows/test.yml

@@ -9,21 +9,18 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.9 ]
+        python-version: [ "3.8", "3.10" ]
     steps:
       - uses: actions/checkout@v1
+      - uses: actions/setup-python@v2
+        name: Set up Python
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install poetry
+        run: pip install poetry
       - name: Install dependencies
-        run: pip install -r requirements.txt
+        run: poetry install
       - name: Test
         run: pytest -svv --cov=bento_drop_box_service --cov-branch
       - name: Codecov
         run: codecov
-  install:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ 3.6, 3.9 ]
-    steps:
-      - uses: actions/checkout@v1
-      - name: Install bento_drop_box_service
-        run: pip install .

Dockerfile

@@ -0,0 +1,29 @@
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-latest
+
+# Use uvicorn (instead of hypercorn) in production since I've found
+# multiple benchmarks showing it to be faster - David L
+RUN pip install --no-cache-dir poetry==1.2.2 "uvicorn[standard]==0.20.0"
+
+# Backwards-compatible with old BentoV2 container layout
+WORKDIR /drop-box
+
+COPY pyproject.toml pyproject.toml
+COPY poetry.toml poetry.toml
+COPY poetry.lock poetry.lock
+
+# Install production dependencies
+# Without --no-root, we get errors related to the code not being copied in yet.
+# But we don't want the code here, otherwise Docker cache doesn't work well.
+RUN poetry install --without dev --no-root
+
+# Manually copy only what's relevant
+# (Don't use .dockerignore, which allows us to have development containers too)
+COPY bento_drop_box_service bento_drop_box_service
+COPY entrypoint.sh entrypoint.sh
+COPY LICENSE LICENSE
+COPY README.md README.md
+
+# Install the module itself, locally (similar to `pip install -e .`)
+RUN poetry install --without dev
+
+CMD [ "sh", "./entrypoint.sh" ]

README.md

@@ -4,7 +4,7 @@
 ![Lint Status](https://github.com/bento-platform/bento_drop_box_service/workflows/Lint/badge.svg)
 [![codecov](https://codecov.io/gh/bento-platform/bento_drop_box_service/branch/master/graph/badge.svg)](https://codecov.io/gh/bento-platform/bento_drop_box_service)
 
-This is a small flask application providing files for ingestion (through `bento_web`,
+This is a small Quart application providing files for ingestion (through `bento_web`,
 for `bento_wes`). By default, the file served are read on the existing filesystem, but
 these can also be read from a minIO instance (or AWS S3 for that matter).
 
@@ -23,30 +23,50 @@ a minIO instance. To do so, you will also need to set `MINIO_USERNAME`,
 
 ## Running in Development
 
-Development dependencies are described in `requirements.txt` and can be
-installed using the following command:
+Poetry is used to manage dependencies.
+
+### Getting set up
+
+1. Create a virtual environment for the project:
+   ```bash
+   virtualenv -p python3 ./env
+   source env/bin/activate
+   ```
+2. Install `poetry`:
+   ```bash
+   pip install poetry
+   ```
+3. Install project dependencies:
+   ```bash
+   poetry install
+   ```
+
+### Running the service locally
+
+To run the service in development mode, use the following command:
 
 ```bash
-pip install -r requirements.txt
+QUART_ENV=development QUART_APP=bento_service_registry.app quart run
 ```
 
-To start the application:
+### Running tests
+
+To run tests and linting, run Tox:
 
 ```bash
-FLASK_APP=bento_drop_box_service.app flask run
+tox
 ```
 
 
-
 ## Deploying
 
 
-The `bento_drop_box_service` service can be deployed with a WSGI server like 
-Gunicorn or UWSGI, specifying `bento_drop_box_service.app:application` as the 
-WSGI application.
+The `bento_drop_box_service` service can be deployed with an ASGI server like 
+Hypercorn, specifying `bento_drop_box_service.app:application` as the 
+ASGI application.
 
 It is best to then put an HTTP server software such as NGINX in front of 
-Gunicorn. 
+Hypercorn. 
 
-**Flask applications should NEVER be deployed in production via the Flask 
-development server, i.e. `flask run`!**
+**Quart applications should NEVER be deployed in production via the Quart 
+development server, i.e. `quart run`!**

bento_drop_box_service/__init__.py

@@ -1,10 +1,6 @@
-import configparser
-import os
+from importlib import metadata
 
 __all__ = ["name", "__version__"]
 
-config = configparser.ConfigParser()
-config.read(os.path.join(os.path.dirname(os.path.realpath(__file__)), "package.cfg"))
-
-name = config["package"]["name"]
-__version__ = config["package"]["version"]
+name = __package__
+__version__ = metadata.version(__package__)

bento_drop_box_service/app.py

@@ -1,13 +1,13 @@
 import os
 
-from bento_lib.responses.flask_errors import (
-    flask_error_wrap,
-    flask_error_wrap_with_traceback,
-    flask_bad_request_error,
-    flask_not_found_error,
-    flask_internal_server_error
+from bento_lib.responses.quart_errors import (
+    quart_error_wrap,
+    quart_error_wrap_with_traceback,
+    quart_bad_request_error,
+    quart_not_found_error,
+    quart_internal_server_error
 )
-from flask import Flask
+from quart import Quart
 from werkzeug.exceptions import BadRequest, NotFound
 
 from bento_drop_box_service.backend import close_backend
@@ -18,24 +18,28 @@
 SERVICE_DATA = os.environ.get("SERVICE_DATA", "data/")
 MINIO_URL = os.environ.get("MINIO_URL", None)
 
-application = Flask(__name__)
+application = Quart(__name__)
 application.config.from_mapping(
+    BENTO_DEBUG=os.environ.get(
+        "CHORD_DEBUG", os.environ.get("BENTO_DEBUG", os.environ.get("QUART_ENV", "production"))
+    ).strip().lower() in ("true", "1", "development"),
     SERVICE_ID=os.environ.get("SERVICE_ID", SERVICE_TYPE),
     SERVICE_DATA_SOURCE="minio" if MINIO_URL else "local",
     SERVICE_DATA=None if MINIO_URL else SERVICE_DATA,
     MINIO_URL=MINIO_URL,
     MINIO_USERNAME=os.environ.get("MINIO_USERNAME") if MINIO_URL else None,
     MINIO_PASSWORD=os.environ.get("MINIO_PASSWORD") if MINIO_URL else None,
     MINIO_BUCKET=os.environ.get("MINIO_BUCKET") if MINIO_URL else None,
+    MINIO_RESOURCE=None,  # manual application-wide override for MinIO boto3 resource
     TRAVERSAL_LIMIT=16,
 )
 
 application.register_blueprint(drop_box_service)
 
 # Generic catch-all
-application.register_error_handler(Exception, flask_error_wrap_with_traceback(flask_internal_server_error,
+application.register_error_handler(Exception, quart_error_wrap_with_traceback(quart_internal_server_error,
                                                                               service_name=SERVICE_NAME))
-application.register_error_handler(BadRequest, flask_error_wrap(flask_bad_request_error))
-application.register_error_handler(NotFound, flask_error_wrap(flask_not_found_error))
+application.register_error_handler(BadRequest, quart_error_wrap(quart_bad_request_error))
+application.register_error_handler(NotFound, quart_error_wrap(quart_not_found_error))
 
 application.teardown_appcontext(close_backend)

bento_drop_box_service/backend.py

@@ -1,18 +1,21 @@
 import os
-from typing import Optional
-from flask import current_app, g
+
+from quart import current_app, g
+
 from .backends.base import DropBoxBackend
 from .backends.local import LocalBackend
 from .backends.minio import MinioBackend
 
+from typing import Optional
+
 
 __all__ = [
     "get_backend",
     "close_backend",
 ]
 
 
-def _get_backend() -> Optional[DropBoxBackend]:
+async def _get_backend() -> Optional[DropBoxBackend]:
     # Make data directory/ies if needed
     if current_app.config["SERVICE_DATA_SOURCE"] == "local":
         os.makedirs(current_app.config["SERVICE_DATA"], exist_ok=True)
@@ -24,13 +27,13 @@ def _get_backend() -> Optional[DropBoxBackend]:
     return None
 
 
-def get_backend() -> Optional[DropBoxBackend]:
+async def get_backend() -> Optional[DropBoxBackend]:
     if "backend" not in g:
-        g.backend = _get_backend()
+        g.backend = await _get_backend()
     return g.backend
 
 
-def close_backend(_e=None):
+async def close_backend(_e=None):
     backend = g.pop("backend", None)
     if backend is not None:
-        backend.close()
+        await backend.close()

bento_drop_box_service/backends/base.py

@@ -8,16 +8,16 @@
 
 class DropBoxBackend(ABC):
     @abstractmethod
-    def get_directory_tree(self) -> Tuple[dict]:
+    async def get_directory_tree(self) -> Tuple[dict]:
         pass
 
     @abstractmethod
-    def upload_to_path(self, request: Request, path: str, content_length: int) -> Response:
+    async def upload_to_path(self, request: Request, path: str, content_length: int) -> Response:
         pass
 
     @abstractmethod
-    def retrieve_from_path(self, path: str) -> Response:
+    async def retrieve_from_path(self, path: str) -> Response:
         pass
 
-    def close(self):
+    async def close(self):
         pass