Merge pull request #525 from jennydaman/gunicorn

Container optimizations in preparation for OpenShift deployment

.gitignore

@@ -14,3 +14,7 @@ celerybeat.pid
 swarm/prod/secrets/
 kubernetes/prod/base/secrets/
 venv
+
+# created by dev container bc UBI sets HOME to the project src dir
+chris_backend/.config
+chris_backend/.bash_history

Dockerfile

@@ -30,52 +30,22 @@
 # docker build --build-arg http_proxy=${PROXY} --build-arg ENVIRONMENT=local -t local/chris:dev .
 #
 
-FROM fnndsc/ubuntu-python3:ubuntu20.04-python3.8.5
+FROM registry.access.redhat.com/ubi9/python-311:1-17.1692772360
 
-LABEL org.opencontainers.image.authors="FNNDSC <dev@babyMRI.org>" \
-      org.opencontainers.image.title="ChRIS Ultron Backend" \
-      org.opencontainers.image.description="ChRIS backend" \
-      org.opencontainers.image.url="https://chrisproject.org/" \
-      org.opencontainers.image.source="https://github.com/FNNDSC/ChRIS_ultron_backEnd" \
-      org.opencontainers.image.licenses="MIT"
-
-# Pass a UID on build command line (see above) to set internal UID
-ARG UID=1001
+COPY --chown=default:root ./requirements/ /tmp/requirements
 ARG ENVIRONMENT=production
-ENV UID=$UID DEBIAN_FRONTEND=noninteractive VERSION="0.1"
-
-ENV APPROOT="/home/localuser/chris_backend" REQPATH="/usr/src/requirements"
-COPY ["./requirements", "${REQPATH}"]
-COPY ["./docker-entrypoint.sh", "/usr/src"]
-
-RUN apt-get update                                               \
-  && apt-get install -y locales                                  \
-  && export LANGUAGE=en_US.UTF-8                                 \
-  && export LANG=en_US.UTF-8                                     \
-  && export LC_ALL=en_US.UTF-8                                   \
-  && locale-gen en_US.UTF-8                                      \
-  && dpkg-reconfigure locales                                    \
-  && apt-get install -y build-essential libldap2-dev libsasl2-dev slapd ldap-utils lcov valgrind \
-  && apt-get install -y libssl-dev libpq-dev                     \
-  && apt-get install -y apache2 apache2-dev                      \
-  && pip install --upgrade pip                                   \
-  && pip install -r ${REQPATH}/${ENVIRONMENT}.txt                \
-  && useradd -l -u $UID -ms /bin/bash localuser
+RUN pip install -r /tmp/requirements/$ENVIRONMENT.txt && rm -rf /tmp/requirements
+COPY chris_backend/ ./
+RUN if [ "$ENVIRONMENT" = "production" ]; then \
+    env DJANGO_SETTINGS_MODULE=config.settings.common ./manage.py collectstatic; fi
 
-# Start as user localuser
-USER localuser
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "4", "config.wsgi:application"]
 
-# Copy source code and make localuser the owner
-COPY --chown=localuser ["./chris_backend", "${APPROOT}"]
-
-WORKDIR $APPROOT
-ENTRYPOINT ["/usr/src/docker-entrypoint.sh"]
-EXPOSE 8000
-
-# Start ChRIS production server
-CMD ["mod_wsgi-express", "start-server", "config/wsgi.py", "--host", "0.0.0.0", "--port", "8000",  \
-"--processes", "4", "--limit-request-body", "5368709120", "--server-root", "/home/localuser/mod_wsgi-0.0.0.0:8000"]
-#to start daemon:
-#/home/localuser/mod_wsgi-0.0.0.0:8000/apachectl start
-#to stop deamon
-#/home/localuser/mod_wsgi-0.0.0.0:8000/apachectl stop
+LABEL org.opencontainers.image.authors="FNNDSC <dev@babyMRI.org>" \
+    org.opencontainers.image.title="ChRIS Backend" \
+    org.opencontainers.image.description="ChRIS backend django API server" \
+    org.opencontainers.image.url="https://chrisproject.org/" \
+    org.opencontainers.image.source="https://github.com/FNNDSC/ChRIS_ultron_backEnd" \
+    org.opencontainers.image.documentation="https://github.com/FNNDSC/ChRIS_ultron_backEnd/wiki/" \
+    org.opencontainers.image.version="" \
+    org.opencontainers.image.licenses="MIT"

chris_backend/config/settings/common.py

@@ -27,7 +27,6 @@
     'django.contrib.staticfiles',
     'django_filters',
     'django_celery_beat',
-    'mod_wsgi.server',
     'rest_framework',
     'rest_framework.authtoken',
     'corsheaders',
@@ -75,6 +74,7 @@
     'core.middleware.ResponseMiddleware',
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.security.SecurityMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
@@ -152,6 +152,11 @@
 
 STATIC_URL = '/static/'
 
+# https://whitenoise.readthedocs.io/en/stable/django.html
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
+
+# `./manage.py collectstatic` is run during build
+STATIC_ROOT = '/opt/app-root/var/staticfiles'
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field

chris_backend/config/settings/local.py

@@ -175,12 +175,12 @@
 
 
 # LDAP auth configuration
-AUTH_LDAP = False
+AUTH_LDAP = True
 if AUTH_LDAP:
-    AUTH_LDAP_SERVER_URI = 'ldap://192.168.0.29:389'
-    AUTH_LDAP_BIND_DN = 'cn=admin,dc=fnndsc,dc=org'
-    AUTH_LDAP_BIND_PASSWORD = 'admin1234'
-    AUTH_LDAP_USER_SEARCH_ROOT = 'dc=fnndsc,dc=org'
+    AUTH_LDAP_SERVER_URI = 'ldap://lldap:3890'
+    AUTH_LDAP_BIND_DN = 'uid=admin,ou=people,dc=example,dc=org'
+    AUTH_LDAP_BIND_PASSWORD = 'chris1234'
+    AUTH_LDAP_USER_SEARCH_ROOT = 'ou=people,dc=example,dc=org'
 
     AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_ROOT, ldap.SCOPE_SUBTREE,
                                        '(uid=%(user)s)')

chris_backend/config/settings/production.py

@@ -119,11 +119,6 @@ def get_secret(setting, secret_type=env):
 }
 
 
-# STATIC FILES (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/4.0/howto/static-files/
-STATIC_ROOT = get_secret('STATIC_ROOT')
-
-
 # CORSHEADERS
 # ------------------------------------------------------------------------------
 CORS_ALLOW_ALL_ORIGINS = get_secret('DJANGO_CORS_ALLOW_ALL_ORIGINS', env.bool)

docker-compose_dev.yml

@@ -29,6 +29,24 @@ services:
     profiles:
       - tools
 
+  db_migrate:
+    image: ${CHRISREPO}/chris:dev
+    build:
+      context: .
+      args:
+        ENVIRONMENT: local
+    volumes:
+      - ./chris_backend:/opt/app-root/src:z
+    environment:
+      - DJANGO_SETTINGS_MODULE=config.settings.local
+    command: python manage.py migrate --noinput
+    user: ${UID}:${GID}
+    depends_on:
+      chris_dev_db:
+        condition: service_healthy
+    networks:
+      local:
+
   chris_dev:
     image: ${CHRISREPO}/chris:dev
     build:
@@ -38,19 +56,22 @@ services:
     stdin_open: true # docker run -i
     tty: true        # docker run -t
     volumes:
-      - ./chris_backend:/home/localuser/chris_backend:z
+      - ./chris_backend:/opt/app-root/src:z
+    user: ${UID}:${GID}
     environment:
       - DJANGO_SETTINGS_MODULE=config.settings.local
-      - DJANGO_DB_MIGRATE=on
-      - DJANGO_COLLECTSTATIC=off
     command: python manage.py runserver 0.0.0.0:8000
     ports:
       - "8000:8000"
     depends_on: 
-      - chris_dev_db
-      - swift_service
-      - queue
-      - chris_store
+      db_migrate:
+        condition: service_completed_successfully
+      swift_service:
+        condition: service_started
+      queue:
+        condition: service_started
+      chris_store:
+        condition: service_started
     networks:
       local:
         aliases:
@@ -71,20 +92,22 @@ services:
       args:
         ENVIRONMENT: local
     volumes:
-      - ./chris_backend:/home/localuser/chris_backend:z
+      - ./chris_backend:/opt/app-root/src:z
+    user: ${UID}:${GID}
     environment:
       - DJANGO_SETTINGS_MODULE=config.settings.local
-      - DJANGO_DB_MIGRATE=off
-      - DJANGO_COLLECTSTATIC=off
       - CELERY_RDB_HOST=0.0.0.0
       - CELERY_RDB_PORT=6900
     command: celery -A core worker -c 3 -l DEBUG -Q main1,main2
     ports:
       - "6900-6905:6900-6905"
     depends_on:
-      - chris_dev_db
-      - swift_service
-      - queue
+      db_migrate:
+        condition: service_completed_successfully
+      swift_service:
+        condition: service_started
+      queue:
+        condition: service_started
       # service also depends on pfcon service defined in swarm/docker-compose_remote.yml
     networks:
       - local
@@ -111,15 +134,18 @@ services:
       args:
         ENVIRONMENT: local
     volumes:
-      - ./chris_backend:/home/localuser/chris_backend:z
+      - ./chris_backend:/opt/app-root/src:z
+    user: ${UID}:${GID}
     environment:
       - DJANGO_SETTINGS_MODULE=config.settings.local
-      - DJANGO_DB_MIGRATE=off
-      - DJANGO_COLLECTSTATIC=off
     command: celery -A core worker -c 1 -l DEBUG -Q periodic
     depends_on:
-      - chris_dev_db
-      - queue
+      db_migrate:
+        condition: service_completed_successfully
+      swift_service:
+        condition: service_started
+      queue:
+        condition: service_started
     networks:
       - local
     labels:
@@ -133,15 +159,18 @@ services:
       args:
         ENVIRONMENT: local
     volumes:
-      - ./chris_backend:/home/localuser/chris_backend:z
+      - ./chris_backend:/opt/app-root/src:z
+    user: ${UID}:${GID}
     environment:
       - DJANGO_SETTINGS_MODULE=config.settings.local
-      - DJANGO_DB_MIGRATE=off
-      - DJANGO_COLLECTSTATIC=off
     command: celery -A core beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
     depends_on:
-      - chris_dev_db
-      - queue
+      db_migrate:
+        condition: service_completed_successfully
+      swift_service:
+        condition: service_started
+      queue:
+        condition: service_started
     # restart until Django DB migrations are ready
     deploy:
       restart_policy:
@@ -165,6 +194,12 @@ services:
     labels:
       name: "ChRIS_ultron_backEnd PostgreSQL Database"
       role: "Backend development database"
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 2s
+      timeout: 4s
+      retries: 3
+      start_period: 60s
 
   queue:
     image: rabbitmq:3
@@ -241,6 +276,22 @@ services:
       name: "Swift"
       role: "Swift object storage service"
 
+  lldap:
+    image: nitnelave/lldap:stable
+    ports:
+      - "3890:3890"
+      - "17170:17170"
+    volumes:
+      - "lldap_data:/data"
+    environment:
+      - UID=10100
+      - GID=10100
+      - TZ=America/New_York
+      - LLDAP_JWT_SECRET=super_secret_random_string
+      - LLDAP_LDAP_USER_PASS=chris1234
+      - LLDAP_LDAP_BASE_DN=dc=example,dc=org
+    networks:
+      local:
 
 networks:
   local:
@@ -254,3 +305,4 @@ volumes:
   chris_store_db_data:
   queue_data:
   swift_storage_dev:
+  lldap_data:

make.sh

@@ -337,6 +337,8 @@ rm -f dc.out ; title -d 1 "Setting global exports"
     boxcenter "-= STOREBASE =-"
     echo "${STOREBASEdisp}"                                               | ./boxes.sh LightCyan
     export STOREBASE=$STOREBASE
+
+    export UID=$(id -u) GID=$(id -g)
 windowBottom
 
 rm -f dc.out ; title -d 1 "Pulling non-'local/' core containers where needed"      \

requirements/base.txt

@@ -3,7 +3,6 @@ django-filter==22.1
 djangorestframework==3.13.1
 django-cors-headers==3.13.0
 psycopg2==2.9.3
-mod-wsgi==4.9.3
 environs==9.5.0
 python-swiftclient==4.1.0
 django-storage-swift==1.2.19
@@ -13,3 +12,4 @@ python-chrisstoreclient==1.0.0
 python-pfconclient==3.2.0
 django-auth-ldap==4.1.0
 PyYAML==6.0.1
+whitenoise[brotli]==6.5.0