From 45635059a3de4b0d999e7e5c16660fff24460e55 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Mon, 11 Sep 2023 13:38:25 -0400 Subject: [PATCH 1/4] Replace mod_wsgi-express with gunicorn in UBI image and remove entrypoint scripts --- .gitignore | 1 + Dockerfile | 59 +++---------- chris_backend/config/settings/common.py | 7 +- chris_backend/config/settings/local.py | 10 +-- chris_backend/config/settings/production.py | 5 -- chris_backend/migratedb.py | 51 ----------- docker-compose_dev.yml | 97 ++++++++++++++++----- docker-entrypoint.sh | 15 ---- make.sh | 2 + requirements/base.txt | 2 +- requirements/production.txt | 4 +- 11 files changed, 105 insertions(+), 148 deletions(-) delete mode 100755 chris_backend/migratedb.py delete mode 100755 docker-entrypoint.sh diff --git a/.gitignore b/.gitignore index 9489d2b4..247f485a 100755 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ celerybeat.pid swarm/prod/secrets/ kubernetes/prod/base/secrets/ venv +chris_backend/.config diff --git a/Dockerfile b/Dockerfile index 5299ad52..9f42cfe9 100755 --- a/Dockerfile +++ b/Dockerfile @@ -30,52 +30,21 @@ # docker build --build-arg http_proxy=${PROXY} --build-arg ENVIRONMENT=local -t local/chris:dev . # -FROM fnndsc/ubuntu-python3:ubuntu20.04-python3.8.5 +FROM registry.access.redhat.com/ubi9/python-311:1-17.1692772360 -LABEL org.opencontainers.image.authors="FNNDSC " \ - org.opencontainers.image.title="ChRIS Ultron Backend" \ - org.opencontainers.image.description="ChRIS backend" \ - org.opencontainers.image.url="https://chrisproject.org/" \ - org.opencontainers.image.source="https://github.com/FNNDSC/ChRIS_ultron_backEnd" \ - org.opencontainers.image.licenses="MIT" - -# Pass a UID on build command line (see above) to set internal UID -ARG UID=1001 +COPY --chown=default:root ./requirements/ /tmp/requirements ARG ENVIRONMENT=production -ENV UID=$UID DEBIAN_FRONTEND=noninteractive VERSION="0.1" - -ENV APPROOT="/home/localuser/chris_backend" REQPATH="/usr/src/requirements" -COPY ["./requirements", "${REQPATH}"] -COPY ["./docker-entrypoint.sh", "/usr/src"] - -RUN apt-get update \ - && apt-get install -y locales \ - && export LANGUAGE=en_US.UTF-8 \ - && export LANG=en_US.UTF-8 \ - && export LC_ALL=en_US.UTF-8 \ - && locale-gen en_US.UTF-8 \ - && dpkg-reconfigure locales \ - && apt-get install -y build-essential libldap2-dev libsasl2-dev slapd ldap-utils lcov valgrind \ - && apt-get install -y libssl-dev libpq-dev \ - && apt-get install -y apache2 apache2-dev \ - && pip install --upgrade pip \ - && pip install -r ${REQPATH}/${ENVIRONMENT}.txt \ - && useradd -l -u $UID -ms /bin/bash localuser +RUN pip install -r /tmp/requirements/$ENVIRONMENT.txt && rm -rf /tmp/requirements +COPY chris_backend/ ./ +RUN env DJANGO_SETTINGS_MODULE=config.settings.common ./manage.py collectstatic -# Start as user localuser -USER localuser +CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "4", "config.wsgi:application"] -# Copy source code and make localuser the owner -COPY --chown=localuser ["./chris_backend", "${APPROOT}"] - -WORKDIR $APPROOT -ENTRYPOINT ["/usr/src/docker-entrypoint.sh"] -EXPOSE 8000 - -# Start ChRIS production server -CMD ["mod_wsgi-express", "start-server", "config/wsgi.py", "--host", "0.0.0.0", "--port", "8000", \ -"--processes", "4", "--limit-request-body", "5368709120", "--server-root", "/home/localuser/mod_wsgi-0.0.0.0:8000"] -#to start daemon: -#/home/localuser/mod_wsgi-0.0.0.0:8000/apachectl start -#to stop deamon -#/home/localuser/mod_wsgi-0.0.0.0:8000/apachectl stop +LABEL org.opencontainers.image.authors="FNNDSC " \ + org.opencontainers.image.title="ChRIS Backend" \ + org.opencontainers.image.description="ChRIS backend django API server" \ + org.opencontainers.image.url="https://chrisproject.org/" \ + org.opencontainers.image.source="https://github.com/FNNDSC/ChRIS_ultron_backEnd" \ + org.opencontainers.image.documentation="https://github.com/FNNDSC/ChRIS_ultron_backEnd/wiki/" \ + org.opencontainers.image.version="" \ + org.opencontainers.image.licenses="MIT" diff --git a/chris_backend/config/settings/common.py b/chris_backend/config/settings/common.py index b7a892da..8ee776a9 100755 --- a/chris_backend/config/settings/common.py +++ b/chris_backend/config/settings/common.py @@ -27,7 +27,6 @@ 'django.contrib.staticfiles', 'django_filters', 'django_celery_beat', - 'mod_wsgi.server', 'rest_framework', 'rest_framework.authtoken', 'corsheaders', @@ -75,6 +74,7 @@ 'core.middleware.ResponseMiddleware', 'corsheaders.middleware.CorsMiddleware', 'django.middleware.security.SecurityMiddleware', + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', @@ -152,6 +152,11 @@ STATIC_URL = '/static/' +# https://whitenoise.readthedocs.io/en/stable/django.html +STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' + +# `./manage.py collectstatic` is run during build +STATIC_ROOT = '/opt/app-root/var/staticfiles' # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field diff --git a/chris_backend/config/settings/local.py b/chris_backend/config/settings/local.py index 5d93f58b..65c42c72 100755 --- a/chris_backend/config/settings/local.py +++ b/chris_backend/config/settings/local.py @@ -175,12 +175,12 @@ # LDAP auth configuration -AUTH_LDAP = False +AUTH_LDAP = True if AUTH_LDAP: - AUTH_LDAP_SERVER_URI = 'ldap://192.168.0.29:389' - AUTH_LDAP_BIND_DN = 'cn=admin,dc=fnndsc,dc=org' - AUTH_LDAP_BIND_PASSWORD = 'admin1234' - AUTH_LDAP_USER_SEARCH_ROOT = 'dc=fnndsc,dc=org' + AUTH_LDAP_SERVER_URI = 'ldap://lldap:3890' + AUTH_LDAP_BIND_DN = 'uid=admin,ou=people,dc=example,dc=org' + AUTH_LDAP_BIND_PASSWORD = 'chris1234' + AUTH_LDAP_USER_SEARCH_ROOT = 'ou=people,dc=example,dc=org' AUTH_LDAP_USER_SEARCH = LDAPSearch(AUTH_LDAP_USER_SEARCH_ROOT, ldap.SCOPE_SUBTREE, '(uid=%(user)s)') diff --git a/chris_backend/config/settings/production.py b/chris_backend/config/settings/production.py index fc223681..0312e90b 100755 --- a/chris_backend/config/settings/production.py +++ b/chris_backend/config/settings/production.py @@ -119,11 +119,6 @@ def get_secret(setting, secret_type=env): } -# STATIC FILES (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/4.0/howto/static-files/ -STATIC_ROOT = get_secret('STATIC_ROOT') - - # CORSHEADERS # ------------------------------------------------------------------------------ CORS_ALLOW_ALL_ORIGINS = get_secret('DJANGO_CORS_ALLOW_ALL_ORIGINS', env.bool) diff --git a/chris_backend/migratedb.py b/chris_backend/migratedb.py deleted file mode 100755 index b79a5112..00000000 --- a/chris_backend/migratedb.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -import time -import sys -import psycopg2 -from argparse import ArgumentParser - -# django needs to be loaded -import django -django.setup() - -from django.core.management import call_command - -from core.models import ChrisInstance - - -parser = ArgumentParser(description="Check database service connection") -parser.add_argument('-u', '--user', help="Database user name") -parser.add_argument('-p', '--password', help="Database user password") -parser.add_argument('-d', '--database', help="Database name") -parser.add_argument('--host', help="Database host") -parser.add_argument('--max-attempts', type=int, dest='attempts', - help="Maximum number of connection attempts") -parser.add_argument('--noinput', action='store_true', - help="Perform migrations in non-interactive mode") - - -# Parse the arguments and perform the appropriate action -args = parser.parse_args() - -host = args.host if args.host else 'localhost' -max_tries = args.attempts if args.attempts else 30 -db = None -while max_tries > 0 and db is None: - try: - db = psycopg2.connect(host=host, user=args.user, password=args.password, - dbname=args.database) - except Exception: - time.sleep(5) - max_tries -= 1 - -if db is None: - print('Could not connect to database service!') - sys.exit(1) -else: - print('Database service ready to accept connections!') - if args.noinput: - call_command("migrate", interactive=False) - else: - call_command("migrate", interactive=True) - ChrisInstance.load() diff --git a/docker-compose_dev.yml b/docker-compose_dev.yml index ca9bd2f0..8b1b2b19 100755 --- a/docker-compose_dev.yml +++ b/docker-compose_dev.yml @@ -29,6 +29,24 @@ services: profiles: - tools + db_migrate: + image: ${CHRISREPO}/chris:dev + build: + context: . + args: + ENVIRONMENT: local + volumes: + - ./chris_backend:/opt/app-root/src:z + environment: + - DJANGO_SETTINGS_MODULE=config.settings.local + command: python manage.py migrate --noinput + user: ${UID}:${GID} + depends_on: + chris_dev_db: + condition: service_healthy + networks: + local: + chris_dev: image: ${CHRISREPO}/chris:dev build: @@ -38,19 +56,23 @@ services: stdin_open: true # docker run -i tty: true # docker run -t volumes: - - ./chris_backend:/home/localuser/chris_backend:z + - ./chris_backend:/opt/app-root/src:z + user: ${UID}:${GID} environment: - DJANGO_SETTINGS_MODULE=config.settings.local - DJANGO_DB_MIGRATE=on - - DJANGO_COLLECTSTATIC=off command: python manage.py runserver 0.0.0.0:8000 ports: - "8000:8000" depends_on: - - chris_dev_db - - swift_service - - queue - - chris_store + db_migrate: + condition: service_completed_successfully + swift_service: + condition: service_started + queue: + condition: service_started + chris_store: + condition: service_started networks: local: aliases: @@ -71,20 +93,22 @@ services: args: ENVIRONMENT: local volumes: - - ./chris_backend:/home/localuser/chris_backend:z + - ./chris_backend:/opt/app-root/src:z + user: ${UID}:${GID} environment: - DJANGO_SETTINGS_MODULE=config.settings.local - - DJANGO_DB_MIGRATE=off - - DJANGO_COLLECTSTATIC=off - CELERY_RDB_HOST=0.0.0.0 - CELERY_RDB_PORT=6900 command: celery -A core worker -c 3 -l DEBUG -Q main1,main2 ports: - "6900-6905:6900-6905" depends_on: - - chris_dev_db - - swift_service - - queue + db_migrate: + condition: service_completed_successfully + swift_service: + condition: service_started + queue: + condition: service_started # service also depends on pfcon service defined in swarm/docker-compose_remote.yml networks: - local @@ -111,15 +135,18 @@ services: args: ENVIRONMENT: local volumes: - - ./chris_backend:/home/localuser/chris_backend:z + - ./chris_backend:/opt/app-root/src:z + user: ${UID}:${GID} environment: - DJANGO_SETTINGS_MODULE=config.settings.local - - DJANGO_DB_MIGRATE=off - - DJANGO_COLLECTSTATIC=off command: celery -A core worker -c 1 -l DEBUG -Q periodic depends_on: - - chris_dev_db - - queue + db_migrate: + condition: service_completed_successfully + swift_service: + condition: service_started + queue: + condition: service_started networks: - local labels: @@ -133,15 +160,18 @@ services: args: ENVIRONMENT: local volumes: - - ./chris_backend:/home/localuser/chris_backend:z + - ./chris_backend:/opt/app-root/src:z + user: ${UID}:${GID} environment: - DJANGO_SETTINGS_MODULE=config.settings.local - - DJANGO_DB_MIGRATE=off - - DJANGO_COLLECTSTATIC=off command: celery -A core beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler depends_on: - - chris_dev_db - - queue + db_migrate: + condition: service_completed_successfully + swift_service: + condition: service_started + queue: + condition: service_started # restart until Django DB migrations are ready deploy: restart_policy: @@ -165,6 +195,12 @@ services: labels: name: "ChRIS_ultron_backEnd PostgreSQL Database" role: "Backend development database" + healthcheck: + test: ["CMD", "pg_isready"] + interval: 2s + timeout: 4s + retries: 3 + start_period: 60s queue: image: rabbitmq:3 @@ -241,6 +277,22 @@ services: name: "Swift" role: "Swift object storage service" + lldap: + image: nitnelave/lldap:stable + ports: + - "3890:3890" + - "17170:17170" + volumes: + - "lldap_data:/data" + environment: + - UID=10100 + - GID=10100 + - TZ=America/New_York + - LLDAP_JWT_SECRET=super_secret_random_string + - LLDAP_LDAP_USER_PASS=chris1234 + - LLDAP_LDAP_BASE_DN=dc=example,dc=org + networks: + local: networks: local: @@ -254,3 +306,4 @@ volumes: chris_store_db_data: queue_data: swift_storage_dev: + lldap_data: \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 2c879d98..00000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -if [[ "$DJANGO_DB_MIGRATE" == 'on' ]]; then - if [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.local' ]]; then - python migratedb.py -u chris -p Chris1234 -d chris_dev --host chris_dev_db --noinput - elif [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.production' ]]; then - python migratedb.py -u $POSTGRES_USER -p $POSTGRES_PASSWORD -d $POSTGRES_DB --host $DATABASE_HOST --noinput - fi -fi - -if [[ "$DJANGO_COLLECTSTATIC" == 'on' ]]; then - python manage.py collectstatic --noinput -fi - -exec "$@" diff --git a/make.sh b/make.sh index 0b25fd83..1e4717c4 100755 --- a/make.sh +++ b/make.sh @@ -337,6 +337,8 @@ rm -f dc.out ; title -d 1 "Setting global exports" boxcenter "-= STOREBASE =-" echo "${STOREBASEdisp}" | ./boxes.sh LightCyan export STOREBASE=$STOREBASE + + export UID=$(id -u) GID=$(id -g) windowBottom rm -f dc.out ; title -d 1 "Pulling non-'local/' core containers where needed" \ diff --git a/requirements/base.txt b/requirements/base.txt index 970729cd..52695206 100755 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,7 +3,6 @@ django-filter==22.1 djangorestframework==3.13.1 django-cors-headers==3.13.0 psycopg2==2.9.3 -mod-wsgi==4.9.3 environs==9.5.0 python-swiftclient==4.1.0 django-storage-swift==1.2.19 @@ -13,3 +12,4 @@ python-chrisstoreclient==1.0.0 python-pfconclient==3.2.0 django-auth-ldap==4.1.0 PyYAML==6.0.1 +whitenoise[brotli]==6.5.0 diff --git a/requirements/production.txt b/requirements/production.txt index b0fe9bb3..8e284f30 100755 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,3 @@ -# Pro-tip: Try not to put anything here. Avoid dependencies in -#production that aren't in development. -r base.txt # includes the base.txt requirements - +gunicorn==21.2.0 From 037a376f9ade6c34944935cc69f99b4ed3bc3402 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Mon, 11 Sep 2023 15:29:36 -0400 Subject: [PATCH 2/4] Remove lingering DJANGO_DB_MIGRATE=on --- docker-compose_dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose_dev.yml b/docker-compose_dev.yml index 8b1b2b19..c27eb9a5 100755 --- a/docker-compose_dev.yml +++ b/docker-compose_dev.yml @@ -60,7 +60,6 @@ services: user: ${UID}:${GID} environment: - DJANGO_SETTINGS_MODULE=config.settings.local - - DJANGO_DB_MIGRATE=on command: python manage.py runserver 0.0.0.0:8000 ports: - "8000:8000" From a87d79b3c0bc387dee850e4719d0bf431970661c Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 14 Sep 2023 09:40:35 -0400 Subject: [PATCH 3/4] Skip collectstatic for dev mode --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f42cfe9..dc408956 100755 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,8 @@ COPY --chown=default:root ./requirements/ /tmp/requirements ARG ENVIRONMENT=production RUN pip install -r /tmp/requirements/$ENVIRONMENT.txt && rm -rf /tmp/requirements COPY chris_backend/ ./ -RUN env DJANGO_SETTINGS_MODULE=config.settings.common ./manage.py collectstatic +RUN if [ "$ENVIRONMENT" = "production" ]; then \ + env DJANGO_SETTINGS_MODULE=config.settings.common ./manage.py collectstatic; fi CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "4", "config.wsgi:application"] From c02308a2519223f875d8c24d8e6cdf3ab669ba8b Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Thu, 14 Sep 2023 09:41:07 -0400 Subject: [PATCH 4/4] Add .bash_history to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 247f485a..8bd8d4bd 100755 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ celerybeat.pid swarm/prod/secrets/ kubernetes/prod/base/secrets/ venv + +# created by dev container bc UBI sets HOME to the project src dir chris_backend/.config +chris_backend/.bash_history