Merge branch 'dev' into rm_defusedxml

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,7 +7,7 @@ assignees: ''
 
 ---
 **Slack us first!**
-The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp-slack.herokuapp.com/)
+The easiest and fastest way to help you is via Slack. There's a free and easy signup to join our #defectdojo channel in the OWASP Slack workspace: [Get Access.](https://owasp.org/slack/invite)
 If you're confident you've found a bug, or are allergic to Slack, you can submit an issue anyway.
 
 **Be informative**

.github/workflows/build-docker-images-for-testing.yml

@@ -35,7 +35,8 @@ jobs:
 
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
+        timeout-minutes: 10
         with:
           context: .
           push: false
@@ -47,6 +48,7 @@ jobs:
 
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
+        timeout-minutes: 10
         uses:  actions/upload-artifact@v3
         with:
           name: ${{ matrix.docker-image }}

.github/workflows/integration-tests.yml

@@ -36,7 +36,7 @@ jobs:
           "tests/tool_config.py",
           "openapi-validatator",
         ]
-        profile: ["mysql-rabbitmq", "postgres-redis"]
+        profile: ["postgres-rabbitmq", "postgres-redis"]
         os: [alpine, debian]
       fail-fast: false
 
@@ -49,6 +49,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              docker load -i nginx/nginx-${{ matrix.os }}_img
              docker load -i django/django-${{ matrix.os }}_img
@@ -58,10 +59,10 @@ jobs:
       - name: Set integration-test mode
         run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
 
-      # phased startup with MySQL and RabbitMQ so we can use the exit code from integrationtest container
-      - name: Start Dojo MySQL + RabbitMQ
-        if: matrix.profile == 'mysql-rabbitmq'
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d mysql nginx celerybeat celeryworker mailhog uwsgi rabbitmq
+      # phased startup with PostgreSQL and RabbitMQ so we can use the exit code from integrationtest container
+      - name: Start Dojo PostgreSQL + RabbitMQ
+        if: matrix.profile == 'postgres-rabbitmq'
+        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi rabbitmq
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
@@ -74,12 +75,14 @@ jobs:
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Initialize
+        timeout-minutes: 10
         run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from initializer initializer
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Integration tests
+        timeout-minutes: 10
         run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from integration-tests integration-tests
         env:
           DD_INTEGRATION_TEST_FILENAME: ${{ matrix.test-case }}

.github/workflows/k8s-tests.yml

@@ -82,6 +82,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              eval $(minikube docker-env)
              docker load -i nginx/nginx-${{ matrix.os }}_img
@@ -103,6 +104,7 @@ jobs:
               echo "rabbit=${{ env.HELM_RABBIT_BROKER_SETTINGS }}" >> $GITHUB_ENV
 
       - name: Deploying Djano application with ${{ matrix.databases }} ${{ matrix.brokers }}
+        timeout-minutes: 10 
         run: |-
              helm install \
                --timeout 800s \
@@ -123,6 +125,7 @@ jobs:
              kubectl get services
 
       - name: Check Application
+        timeout-minutes: 10
         run: |-
              to_complete () {
                kubectl wait --for=$1 $2 --timeout=500s --selector=$3 2>/tmp/test || true

.github/workflows/release-x-manual-docker-containers.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Build and push images with debian
         if: ${{ matrix.os  == 'debian' }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           REPO_ORG: ${{ env.repoorg }}
           docker-image: ${{ matrix.docker-image }}
@@ -77,7 +77,7 @@ jobs:
 
       - name: Build and push images with alpine
         if: ${{ matrix.os  == 'alpine' }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         env:
           REPO_ORG: ${{ env.repoorg }}
           docker-image: ${{ matrix.docker-image }}

.github/workflows/rest-framework-tests.yml

@@ -23,6 +23,7 @@ jobs:
         uses: actions/download-artifact@v3
 
       - name: Load docker images
+        timeout-minutes: 10
         run: |-
              docker load -i nginx/nginx-${{ matrix.os }}_img
              docker load -i django/django-${{ matrix.os }}_img
@@ -33,19 +34,20 @@ jobs:
         run: docker/setEnv.sh unit_tests_cicd
 
       # phased startup so we can use the exit code from unit test container
-      - name: Start MySQL
-        run: docker compose --env-file ./docker/environments/mysql-redis.env up -d mysql
+      - name: Start Postgres
+        run: docker compose --env-file ./docker/environments/postgres-redis.env up -d postgres
 
       # no celery or initializer needed for unit tests
       - name: Unit tests
-        run: docker compose --profile mysql-redis --env-file ./docker/environments/mysql-redis.env up --no-deps --exit-code-from uwsgi uwsgi
+        timeout-minutes: 10
+        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps --exit-code-from uwsgi uwsgi
         env:
           DJANGO_VERSION: ${{ matrix.os }}
 
       - name: Logs
         if: failure()
-        run: docker compose --profile mysql-redis --env-file ./docker/environments/mysql-redis.env logs --tail="2500" uwsgi
+        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env logs --tail="2500" uwsgi
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile mysql-redis --env-file ./docker/environments/mysql-redis.env down
+        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env down

Dockerfile.django-alpine

@@ -5,7 +5,7 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.3-alpine3.16@sha256:9efc6e155f287eb424ede74aeff198be75ae04504b1e42e87ec9f221e7410f2d as base
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
 FROM base as build
 WORKDIR /app
 RUN \
@@ -76,6 +76,7 @@ COPY \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.django-debian

@@ -5,7 +5,7 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.4-slim-bullseye@sha256:40319d0a897896e746edf877783ef39685d44e90e1e6de8d964d0382df0d4952 as base
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as base
 FROM base as build
 WORKDIR /app
 RUN \
@@ -43,10 +43,10 @@ RUN \
   # ugly fix to install postgresql-client without errors
   mkdir -p /usr/share/man/man1 /usr/share/man/man7 && \
   apt-get -y install --no-install-recommends \
-    # libopenjp2-7 libjpeg62 libtiff5 are required by the pillow package
+    # libopenjp2-7 libjpeg62 libtiff are required by the pillow package
     libopenjp2-7 \
     libjpeg62 \
-    libtiff5 \
+    libtiff6 \
     dnsutils \
     default-mysql-client \
     libmariadb3 \
@@ -81,6 +81,7 @@ COPY \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.integration-tests-debian

@@ -2,7 +2,7 @@
 # code: language=Dockerfile
 
 FROM openapitools/openapi-generator-cli:v7.6.0@sha256:f86ca824293602b71b9b66683cc0011f8ff963858bd853621c554ff5cc7dd1d5 as openapitools
-FROM python:3.11.4-slim-bullseye@sha256:40319d0a897896e746edf877783ef39685d44e90e1e6de8d964d0382df0d4952 as build
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -62,6 +62,7 @@ COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/ta
 
 COPY docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
+  docker/reach_database.sh \
   docker/entrypoint-integration-tests.sh \
   /
 

Dockerfile.nginx-alpine

@@ -5,7 +5,7 @@
 # Dockerfile.django-alpine to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.3-alpine3.16@sha256:9efc6e155f287eb424ede74aeff198be75ae04504b1e42e87ec9f221e7410f2d as base
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
 FROM base as build
 WORKDIR /app
 RUN \
@@ -140,7 +140,7 @@ COPY manage.py ./
 COPY dojo/ ./dojo/
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.26.0-alpine@sha256:ef587d1eb99e991291c582bfb74f27db27f7ca2c095d4ba06cc3f7c910a0c7b3
+FROM nginx:1.27.0-alpine@sha256:a45ee5d042aaa9e81e013f97ae40c3dda26fbe98f22b6251acdf28e579560d55
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

Dockerfile.nginx-debian

@@ -5,7 +5,7 @@
 # Dockerfile.django-debian to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.4-slim-bullseye@sha256:40319d0a897896e746edf877783ef39685d44e90e1e6de8d964d0382df0d4952 as base
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as base
 FROM base as build
 WORKDIR /app
 RUN \
@@ -75,7 +75,7 @@ COPY dojo/ ./dojo/
 
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.26.0-alpine@sha256:ef587d1eb99e991291c582bfb74f27db27f7ca2c095d4ba06cc3f7c910a0c7b3
+FROM nginx:1.27.0-alpine@sha256:a45ee5d042aaa9e81e013f97ae40c3dda26fbe98f22b6251acdf28e579560d55
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "defectdojo",
-  "version": "2.35.0-dev",
+  "version": "2.36.0-dev",
   "license" : "BSD-3-Clause",
   "private": true,
   "dependencies": {

docker-compose.yml

@@ -54,6 +54,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -75,6 +76,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -95,6 +97,7 @@ services:
       DD_CELERY_BROKER_URL: ${DD_CELERY_BROKER_URL}
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -119,6 +122,7 @@ services:
       DD_INITIALIZE: "${DD_INITIALIZE:-true}"
       DD_SECRET_KEY: "${DD_SECRET_KEY:-hhZCp@D28z!n@NED*yB!ROMt+WzsY*iq}"
       DD_CREDENTIAL_AES_256_KEY: "${DD_CREDENTIAL_AES_256_KEY:-&91a*agLqesc*0DJ+2*bAbsUZfR*4nLw}"
+      DD_DATABASE_READINESS_TIMEOUT: "${DD_DATABASE_READINESS_TIMEOUT:-30}"
     volumes:
         - type: bind
           source: ./docker/extra_settings
@@ -137,7 +141,7 @@ services:
     volumes:
       - defectdojo_data:/var/lib/mysql
   postgres:
-    image: postgres:16.3-alpine@sha256:e89da2c083a5405943408b6807cd1fd25dc9010c1294e30611b841778bedc653
+    image: postgres:16.3-alpine@sha256:de3d7b6e4b5b3fe899e997579d6dfe95a99539d154abe03f0b6839133ed05065
     profiles: 
       - postgres-rabbitmq
       - postgres-redis
@@ -155,7 +159,7 @@ services:
     volumes:
        - defectdojo_rabbitmq:/var/lib/rabbitmq
   redis:
-    image: redis:7.2.5-alpine@sha256:0389bb8416d7c6ed065c25745179bf5d358e5d9472dd30a687ab36ffbb650262
+    image: redis:7.2.5-alpine@sha256:de14eedfbd1fc871d0f5aa1773fd80743930e45354d035b6f3b551e7ffa44df8
     profiles: 
       - mysql-redis
       - postgres-redis

docker/entrypoint-celery-beat.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+
+. /reach_database.sh
+
 umask 0002
 
 id
@@ -16,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
     rm -f /app/dojo/settings/README.md
 fi
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 # do the check with Django stack

docker/entrypoint-celery-worker.sh

@@ -4,6 +4,7 @@ umask 0002
 id
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null)
@@ -18,12 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
     rm -f /app/dojo/settings/README.md
 fi
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

docker/entrypoint-initializer.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 initialize_data()
 {
@@ -60,12 +61,7 @@ then
 fi
 echo "Initializing."
 
-echo -n "Waiting for database to be reachable "
-until echo "select 1;" | python3 manage.py dbshell > /dev/null
-do
-  echo -n "."
-  sleep 1
-done
+wait_for_database_to_be_reachable
 echo
 
 echo "Checking ENABLE_AUDITLOG"

docker/entrypoint-unit-tests-devDocker.sh

@@ -7,6 +7,7 @@ set -e
 set -v
 
 . /secret-file-loader.sh
+. /reach_database.sh
 
 cd /app
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
@@ -15,6 +16,8 @@ unset DD_DATABASE_URL
 # Unset the celery broker URL so that we can force the other DD_CELERY_BROKER settings
 unset DD_CELERY_BROKER_URL
 
+wait_for_database_to_be_reachable
+
 python3 manage.py makemigrations dojo
 python3 manage.py migrate
 
@@ -50,7 +53,7 @@ EOF
 
 echo "Unit Tests"
 echo "------------------------------------------------------------"
-python3 manage.py test unittests -v 3 --keepdb --no-input
+python3 manage.py test unittests -v 3 --keepdb --no-input --shuffle
 
 # you can select a single file to "test" unit tests
 # python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3