AWS S3 backend storage integration 🍉

Dockerfile

@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright (c) 2021-present Kaleidos Ventures SL
+
+FROM python:3.11-slim
+LABEL maintainer="support@taiga.io"
+
+# Avoid prompting for configuration
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONFAULTHANDLER=1
+
+# Use a virtualenv
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Get the code
+COPY . /taiga-back
+WORKDIR /taiga-back
+
+# grab gosu for easy step-down from root
+# https://github.com/tianon/gosu/blob/master/INSTALL.md
+ENV GOSU_VERSION 1.12
+
+RUN set -eux; \
+    apt-get update; \
+    # install system dependencies
+    apt-get install -y \
+       build-essential \
+       gettext \
+       # libpq5 needed in runtime for psycopg2
+       libpq5 \
+       libpq-dev \
+       git \
+       net-tools \
+       procps \
+       wget; \
+    # install gosu
+    apt-get install -y --no-install-recommends ca-certificates wget; \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
+    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
+	wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
+    chmod +x /usr/local/bin/gosu; \
+    # verify gosu signature
+    export GNUPGHOME="$(mktemp -d)"; \
+	gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
+	gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
+	command -v gpgconf && gpgconf --kill all || :; \
+	rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
+    # install Taiga dependencies
+    python -m pip install --upgrade pip; \
+    python -m pip install wheel; \
+    python -m pip install -r requirements.txt; \
+    python -m pip install -r requirements-contribs.txt; \
+    python manage.py compilemessages; \
+    python manage.py collectstatic --no-input; \
+    chmod +x docker/entrypoint.sh; \
+    chmod +x docker/async_entrypoint.sh; \
+    cp docker/config.py settings/config.py; \
+    #  create taiga group and user to use it and give permissions over the code (in entrypoint)
+    groupadd --system taiga --gid=999; \
+    useradd --system --no-create-home --gid taiga --uid=999 --shell=/bin/bash taiga; \
+    mkdir -p /taiga-back/media/exports; \
+    chown -R taiga:taiga /taiga-back; \
+    # remove unneeded files and packages
+    apt-get purge -y \
+       build-essential \
+       gettext \
+       git \
+       libpq-dev \
+       net-tools \
+       procps \
+       wget; \
+    apt-get autoremove -y; \
+    rm -rf /var/lib/apt/lists/*; \
+    rm -rf /root/.cache; \
+    # clean taiga
+    rm requirements.txt; \
+    rm requirements-contribs.txt; \
+    find . -name '__pycache__' -exec rm -r '{}' +; \
+    find . -name '*pyc' -exec rm -r '{}' +; \
+    find . -name '*po' -exec rm -r '{}' +
+
+ENV DJANGO_SETTINGS_MODULE=settings.config
+
+EXPOSE 8000
+ENTRYPOINT ["./docker/entrypoint.sh"]

Pipfile

@@ -0,0 +1,97 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+aggdraw = "==1.3.15"
+amqp = "==5.1.1"
+asana = "==3.1.0"
+asgiref = "==3.6.0"
+async-timeout = "==4.0.2"
+attrs = "==22.2.0"
+backoff = "==1.6.0"
+billiard = "==3.6.4.0"
+bleach = "==4.1.0"
+cairocffi = "==1.4.0"
+cairosvg = "==2.6.0"
+celery = "==5.2.7"
+certifi = "==2022.12.7"
+cffi = "==1.15.1"
+charset-normalizer = "==2.0.12"
+click = "==8.1.3"
+click-didyoumean = "==0.3.0"
+click-plugins = "==1.1.1"
+click-repl = "==0.2.0"
+cryptography = "==39.0.1"
+cssselect = "==1.2.0"
+cssselect2 = "==0.7.0"
+cssutils = "==2.6.0"
+defusedxml = "==0.7.1"
+diff-match-patch = "==20121119"
+django = "==3.2.19"
+django-ipware = "==1.1.6"
+django-jinja = "==2.10.2"
+django-pglocks = "==1.0.4"
+django-picklefield = "==3.1"
+django-sampledatahelper = "==0.4.1"
+django-sites = "==0.11"
+django-sr = "==0.0.4"
+djmail = "==2.0.0"
+django-storages = "*"
+boto3 = "==1.27.0"
+docopt = "==0.6.2"
+easy-thumbnails = "==2.8.5"
+gunicorn = "==20.1.0"
+html5lib = "==1.1"
+idna = "==3.4"
+imageio = "==2.25.1"
+importlib-metadata = "==6.0.0"
+jinja2 = "==3.1.2"
+kombu = "==5.2.4"
+lxml = "==4.9.2"
+markdown = "==3.4.1"
+markupsafe = "==2.1.2"
+monotonic = "==1.6"
+netaddr = "==0.8.0"
+networkx = "==3.0"
+numpy = "==1.24.2"
+oauthlib = {version = "==3.2.2", extras = ["signedtoken"]}
+pillow = "==9.4.0"
+premailer = "==3.0.1"
+prompt-toolkit = "==3.0.36"
+psd-tools = "==1.9.18"
+psycopg2 = "==2.9.5"
+pycparser = "==2.21"
+pygments = "==2.14.0"
+pyjwt = "==2.6.0"
+pymdown-extensions = "==9.9.2"
+python-dateutil = "==2.7.5"
+python-magic = "==0.4.15"
+pytz = "==2022.7.1"
+pywavelets = "==1.4.1"
+raven = "==6.10.0"
+redis = "==4.5.3"
+requests = "==2.27.1"
+requests-oauthlib = "==1.3.1"
+rudder-sdk-python = "==1.0.0b1"
+sampledata = "==0.3.7"
+scikit-image = "==0.19.3"
+scipy = "==1.10.0"
+serpy = "==0.1.1"
+six = "==1.16.0"
+sqlparse = "==0.4.3"
+tifffile = "==2023.2.3"
+tinycss2 = "==1.2.1"
+unidecode = "==0.4.20"
+urllib3 = "==1.26.14"
+vine = "==5.0.0"
+wcwidth = "==0.2.6"
+webcolors = "==1.9.1"
+webencodings = "==0.5.1"
+zipp = "==1.2.0"
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"

docker/config.py

@@ -50,6 +50,9 @@
 #########################################
 MEDIA_URL = f"{ TAIGA_URL }/media/"
 DEFAULT_FILE_STORAGE = "taiga_contrib_protected.storage.ProtectedFileSystemStorage"
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
+if AWS_STORAGE_BUCKET_NAME:
+    DEFAULT_FILE_STORAGE = "settings.storage_backend.PrivateMediaStorage"
 THUMBNAIL_DEFAULT_STORAGE = DEFAULT_FILE_STORAGE
 
 STATIC_URL = f"{ TAIGA_URL }/static/"

requirements.in

@@ -11,6 +11,7 @@ django-sites==0.11
 django-sr==0.0.4
 djmail==2.0.0
 easy-thumbnails==2.8.5
+django-storages==1.13.2
 gunicorn==20.1.0
 netaddr<0.9
 premailer==3.0.1

requirements.txt

@@ -71,20 +71,22 @@ django==3.2.19
     #   easy-thumbnails
 django-ipware==1.1.6
     # via -r requirements.in
-django-jinja==2.10.2
+django-jinja
     # via -r requirements.in
 django-pglocks==1.0.4
     # via -r requirements.in
 django-picklefield==3.1
     # via -r requirements.in
-django-sampledatahelper==0.4.1
+django-sampledatahelper
     # via -r requirements.in
 django-sites==0.11
     # via -r requirements.in
-django-sr==0.0.4
+django-sr
     # via -r requirements.in
 djmail==2.0.0
     # via -r requirements.in
+django-storages
+boto3==1.27.0
 docopt==0.6.2
     # via psd-tools
 easy-thumbnails==2.8.5
@@ -93,7 +95,7 @@ gunicorn==20.1.0
     # via -r requirements.in
 html5lib==1.1
     # via -r requirements.in
-idna==3.4
+idna
     # via requests
 imageio==2.25.1
     # via scikit-image
@@ -175,7 +177,7 @@ raven==6.10.0
     # via -r requirements.in
 redis==4.5.3
     # via -r requirements.in
-requests==2.27.1
+requests
     # via
     #   -r requirements.in
     #   asana
@@ -220,7 +222,7 @@ tinycss2==1.2.1
     #   cssselect2
 unidecode==0.4.20
     # via -r requirements.in
-urllib3==1.26.14
+urllib3
     # via requests
 vine==5.0.0
     # via

settings/common.py

@@ -683,3 +683,25 @@
     print("Try: \033[1;33mpy.test\033[0m")
     sys.exit(0)
 # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE
+
+
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
+if AWS_STORAGE_BUCKET_NAME:
+    del STATIC_ROOT
+    AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
+
+    AWS_S3_CUSTOM_DOMAIN = f"{AWS_STORAGE_BUCKET_NAME}.s3.amazonaws.com"
+    AWS_S3_OBJECT_PARAMETERS = {    
+        "CacheControl": "max-age=86400",
+    }
+    AWS_LOCATION = "static"
+    AWS_S3_SIGNATURE_VERSION = "s3v4"
+
+    STATICFILES_DIRS = [
+        os.path.join(BASE_DIR, "static"),
+    ]
+
+    STATIC_URL = f"https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_LOCATION}/"
+    STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+    DEFAULT_FILE_STORAGE = "settings.storage_backend.PrivateMediaStorage"

settings/storage_backend.py

@@ -0,0 +1,13 @@
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class MediaStorage(S3Boto3Storage):
+    location = 'media'
+    file_overwrite = True
+
+
+class PrivateMediaStorage(S3Boto3Storage):
+    location = "private"
+    default_acl = "private"
+    file_overwrite = False
+    custom_domain = False