Merge pull request #524 from internetstandards/50

Version 5.0.0

Dockerfile

@@ -35,6 +35,8 @@ USER root
 ENV UWSGI_GID root
 ENV UWSGI_UID root
 ENV UWSGI_MODULE dashboard.wsgi
+# Fix uwsgi "invalid cheaper value: must be lower than processes", see #495.
+ENV UWSGI_CHEAPER 0
 # serve static files (to caching proxy) from collected/generated static files
 ENV UWSGI_STATIC_MAP /static=/srv/dashboard/static
 # set proxy and browser caching for static files to 1 month

Makefile

@@ -132,6 +132,7 @@ pylama: ${pysrc} ${app}
 	# check code quality
 	${env} pylama ${pysrcdirs} --skip "**/migrations/*"
 
+
 shellcheck: ${shsrc}
 	# shell script checks (if installed)
 	if command -v shellcheck &>/dev/null && ! test -z "${shsrc}";then ${env} shellcheck ${shsrc}; fi
@@ -143,6 +144,7 @@ autofix fix: ${pysrc} ${app} ## automatic fix of trivial code quality issues
 	${env} autoflake -ri --remove-all-unused-imports ${pysrcdirs}
 	# sort imports
 	${env} isort -rc ${pysrcdirs}
+	black .
 	# do a check after autofixing to show remaining problems
 	${MAKE} check
 

README.md

@@ -1,3 +1,6 @@
+For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/50/docs/render/markdown/1_installation.md) 
+and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/50/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
+
 # Internet.nl Dashboard
 The internet.nl dashboard allows you to visualize batch scans from the internet.nl API. It allows:
 
@@ -13,6 +16,12 @@ The internet.nl dashboard allows you to visualize batch scans from the internet.
 - Second factor authentication
 - Seamless internet.nl API integration
 
+## Setup / installation
+
+For quick installation: Follow [these quick instructions](https://github.com/internetstandards/Internet.nl-dashboard/blob/50/docs/render/markdown/1_installation.md) 
+and watch [this 6 minute video](https://github.com/internetstandards/Internet.nl-dashboard/tree/50/docs/input/internet.nl%20dashboard%20installation%20video%20small.mp4).
+
+
 ## Screenshots
 
 Some screenshots can be found at: https://dashboard.internet.nl/#/tour

compose.yaml

@@ -0,0 +1,124 @@
+services:
+  backend:
+    image: ${DOCKER_IMAGE_DASHBOARD:-${DOCKER_REGISTRY:-internetstandards}/dashboard:${RELEASE:-latest}}
+    build: .
+    command: dashboard_prdserver --migrate
+    # TODO: check if needed: --security-opt seccomp=unconfined
+    environment:
+      SECRET_KEY_FILE: /etc/django_keys/secret_key
+      FIELD_ENCRYPTION_KEY_FILE: /etc/django_keys/field_encryption_key
+      ALLOWED_HOSTS: "*"
+      UWSGI_HARAKIRI: 3600
+      DJANGO_DATABASE: production
+      DB_ENGINE: postgresql_psycopg2
+      DB_HOST: database
+      WORKER_ROLE: default
+      BROKER: redis://broker:6379/0
+      SENTRY_DSN:
+      DASHBOARD_FRONTEND_URL:
+    networks:
+      dashboard:
+    volumes:
+      - dashboard-frontend-uploads:/source/dashboard/uploads
+      - django-keys:/etc/django_keys/
+    healthcheck:
+      interval: 60s
+
+  frontend:
+    image: ${DOCKER_IMAGE_DASHBOARD_STATIC:-${DOCKER_REGISTRY:-internetstandards}/dashboard-static:${RELEASE:-latest}}
+    build: https://github.com/internetstandards/Internet.nl-dashboard-frontend.git
+    networks:
+      dashboard:
+    healthcheck:
+      interval: 60s
+    volumes:
+      - dashboard-frontend-uploads:/source/dashboard/uploads
+
+  worker: &worker
+    image: ${DOCKER_IMAGE_DASHBOARD:-${DOCKER_REGISTRY:-internetstandards}/dashboard:${RELEASE:-latest}}
+    build: .
+    command: celery_dashboard worker --queues storage,celery,isolated,kickoff,kickoff1,kickoff2,kickoff3,kickoff4,database_deprecate,database_deprecate3,database,database3
+    environment: &worker-environment
+      SECRET_KEY_FILE: /etc/django_keys/secret_key
+      FIELD_ENCRYPTION_KEY_FILE: /etc/django_keys/field_encryption_key
+      DJANGO_DATABASE: production
+      DB_ENGINE: postgresql_psycopg2
+      DB_HOST: database
+      WORKER_ROLE: default_ipv4
+      BROKER: redis://broker:6379/0
+      SENTRY_DSN:
+    networks:
+      dashboard:
+    healthcheck:
+      interval: 60s
+    volumes:
+      - django-keys:/etc/django_keys/
+      - dashboard-frontend-uploads:/source/dashboard/uploads
+
+  worker-reporting:
+    <<: *worker
+    command: celery_dashboard worker --queues reporting
+    environment:
+      <<: *worker-environment
+      WORKER_ROLE: reporting
+
+  worker-scanning:
+    <<: *worker
+    command: celery_dashboard worker --queues ipv4,internet
+    environment:
+      <<: *worker-environment
+      WORKER_ROLE: default_ipv4
+
+  scheduler:
+    <<: *worker
+    image: ${DOCKER_IMAGE_DASHBOARD:-${DOCKER_REGISTRY:-internetstandards}/dashboard:${RELEASE:-latest}}
+    build: .
+    command: celery_dashboard beat -l info --pidfile=/var/tmp/celerybeat.pid
+
+  webserver:
+    image: caddy
+    configs:
+      - source: caddyfile
+        target: /etc/caddy/Caddyfile
+    ports:
+      - 127.0.0.1:8000:8000
+    networks:
+      dashboard:
+
+  database:
+    image: postgres:12
+    environment:
+      POSTGRES_DB: dashboard
+      POSTGRES_USER: dashboard
+      POSTGRES_PASSWORD: dashboard
+    networks:
+      dashboard:
+    volumes:
+      - database:/var/lib/postgresql/data
+    healthcheck:
+      interval: 60s
+    stop_grace_period: 60s
+
+  broker:
+    image: redis
+    healthcheck:
+      interval: 60s
+    networks:
+      dashboard:
+
+configs:
+  caddyfile:
+    content: |
+      :8000 {
+        @paths path /account/* /admin/* /data/* /jet/* /logout/* /mail/* /session/* /static/* /upload/* /security.txt /.well-known/security.txt
+        reverse_proxy @paths backend:8000
+        reverse_proxy /* frontend:80
+      }
+
+volumes:
+  dashboard-frontend-uploads:
+  database:
+  django-keys:
+
+networks:
+  dashboard:

dashboard/celery/__init__.py

@@ -21,8 +21,9 @@
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
 
 # autodiscover all celery tasks in tasks.py files inside websecmap modules
-app.autodiscover_tasks([app for app in settings.INSTALLED_APPS
-                        if app.startswith('dashboard') or app.startswith('websecmap')])
+app.autodiscover_tasks(
+    [app for app in settings.INSTALLED_APPS if app.startswith("dashboard") or app.startswith("websecmap")]
+)
 
 # http://docs.celeryproject.org/en/master/whatsnew-4.0.html?highlight=priority#redis-priorities-reversed
 # http://docs.celeryproject.org/en/master/history/whatsnew-3.0.html?highlight=priority
@@ -31,9 +32,9 @@
 # https://github.com/celery/celery/blob/f83b072fba7831f60106c81472e3477608baf289/docs/whatsnew-4.0.rst#redis-priorities-reversed
 # contrary to 'documentation' in release notes the redis priorities do not seem aligned with rabbitmq
 app.conf.broker_transport_options = {
-    'priority_steps': [1, 5, 9],
+    "priority_steps": [1, 5, 9],
 }
-if 'redis://' in app.conf.broker_url:
+if "redis://" in app.conf.broker_url:
     PRIO_HIGH = 1
     PRIO_NORMAL = 5
     PRIO_LOW = 9
@@ -44,8 +45,8 @@
 
 # lookup table for routing keys for different IP versions
 IP_VERSION_QUEUE = {
-    4: 'scanners.ipv4',
-    6: 'scanners.ipv6',
+    4: "scanners.ipv4",
+    6: "scanners.ipv6",
 }
 
 
@@ -77,21 +78,24 @@ def status():
     active = inspect.active()
     reserved = inspect.reserved()
     active_queues = inspect.active_queues()
-    workers = [{
-        'name': worker_name,
-        'queues': [q['name'] for q in active_queues.get(worker_name, [])],
-        'tasks_processed': sum(worker_stats['total'].values()),
-        'tasks_active': len(active.get(worker_name, [])),
-        'tasks_reserved': len(reserved.get(worker_name, [])),
-        'prefetch_count': worker_stats['prefetch_count'],
-        'concurrency': worker_stats['pool']['max-concurrency'],
-    } for worker_name, worker_stats in stats.items()]
+    workers = [
+        {
+            "name": worker_name,
+            "queues": [q["name"] for q in active_queues.get(worker_name, [])],
+            "tasks_processed": sum(worker_stats["total"].values()),
+            "tasks_active": len(active.get(worker_name, [])),
+            "tasks_reserved": len(reserved.get(worker_name, [])),
+            "prefetch_count": worker_stats["prefetch_count"],
+            "concurrency": worker_stats["pool"]["max-concurrency"],
+        }
+        for worker_name, worker_stats in stats.items()
+    ]
 
     # todo: fix Returning Any from function declared to return "SupportsLessThan"
-    workers = sorted(workers, key=lambda k: (k['name']), reverse=False)  # type: ignore
+    workers = sorted(workers, key=lambda k: (k["name"]), reverse=False)  # type: ignore
 
-    if 'redis://' in app.conf.broker_url:
-        queue_names = [q.name for q in QUEUES_MATCHING_ROLES['queuemonitor']]
+    if "redis://" in app.conf.broker_url:
+        queue_names = [q.name for q in QUEUES_MATCHING_ROLES["queuemonitor"]]
 
         # on localhost and remote workers there is no event loop. This causes an exception.
         # Inspired on https://github.com/tornadoweb/tornado/issues/2352 and
@@ -103,6 +107,7 @@ def status():
         # 'solves': RuntimeError: There is no current event loop in thread 'Thread-3'.
         try:
             import asyncio  # pylint: disable=import-outside-toplevel
+
             asyncio.set_event_loop(asyncio.new_event_loop())
         except BaseException:  # pylint: disable=broad-except
             # an eventloop already exists.
@@ -118,21 +123,17 @@ def status():
             log.error("Could not connect to flower to retrieve queue stats.")
             log.exception(runtime_error)
 
-        queues = [{'name': x['name'], 'tasks_pending': x['messages']} for x in queue_stats]
+        queues = [{"name": x["name"], "tasks_pending": x["messages"]} for x in queue_stats]
     else:
-        raise NotImplementedError('Currently only Redis is supported!')
+        raise NotImplementedError("Currently only Redis is supported!")
 
     # todo: fix Returning Any from function declared to return "SupportsLessThan"
-    queues = sorted(queues, key=lambda k: (k['name']), reverse=False)  # type: ignore
+    queues = sorted(queues, key=lambda k: (k["name"]), reverse=False)  # type: ignore
 
     alerts = []
     if not workers:
-        alerts.append('No active workers!')
+        alerts.append("No active workers!")
     if len(workers) > 9000:
-        alerts.append('Number of workers is OVER 9000!!!!1111')
+        alerts.append("Number of workers is OVER 9000!!!!1111")
 
-    return {
-        'alerts': alerts,
-        'workers': workers,
-        'queues': queues
-    }
+    return {"alerts": alerts, "workers": workers, "queues": queues}

dashboard/internet_nl_dashboard/__init__.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
-default_app_config = 'dashboard.internet_nl_dashboard.apps.DashboardConfig'  # pylint: disable=invalid-name
+default_app_config = "dashboard.internet_nl_dashboard.apps.DashboardConfig"  # pylint: disable=invalid-name
 
 log = logging.getLogger(__package__)