feat: add docker env

.gitignore

@@ -139,3 +139,10 @@ secret.py
 # PyCharm
 .idea/
 
+/manager/director-shell-keys/
+/orchestrator/director-shell-keys/
+/mysql-data/
+/postgres-data/
+/postgres-manager-data/
+/redis-data/
+/registry-data/

config/Dockerfile.manager

@@ -0,0 +1,28 @@
+FROM python:3.8.10
+
+RUN groupadd director && \
+    useradd -g director director && \
+    usermod -a -G director www-data && \
+    mkdir -p /var/www/director && \
+    cd /var/www/director && \
+    chown director:director /var/www/director && \
+    chmod 750 /var/www/director
+
+USER director
+
+WORKDIR /var/www/director
+
+COPY manager/requirements.txt manager/
+
+RUN pip install -r manager/requirements.txt
+
+COPY scripts/ manager/scripts/
+RUN mkdir -p /etc/director-shell-keys/ && \
+    ./manager/scripts/generate-rsa-key.py 4096 /etc/director-shell-keys/shell-signing-token-pubkey.pem /etc/director-shell-keys/shell-signing-token-privkey.pem && \
+    ./manager/scripts/generate-rsa-key.py 4096 /etc/director-shell-keys/shell-encryption-token-pubkey.pem /etc/director-shell-keys/shell-encryption-token-privkey.pem
+
+COPY manager/ /var/www/director/manager/
+
+ENV PYTHONUNBUFFERED=1
+
+ENTRYPOINT ["sh", "start.sh"]

config/Dockerfile.orchestrator

@@ -0,0 +1,30 @@
+FROM python:3.8.10
+
+RUN apt-get update && apt-get install -y supervisor
+RUN mkdir -p /var/log/supervisor
+
+COPY orchestrator/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY orchestrator/requirements.txt orchestrator/
+
+RUN groupadd director && \
+    useradd -g director director && \
+    usermod -a -G director www-data && \
+    mkdir -p /var/www/director && \
+    cd /var/www/director && \
+    chown director:director /var/www/director && \
+    chmod 750 /var/www/director && \
+    pip install -r orchestrator/requirements.txt
+
+USER director
+
+WORKDIR /var/www/director
+
+# RUN pip install -r orchestrator/requirements.txt
+
+COPY scripts/ orchestrator/scripts/
+
+COPY orchestrator/ orchestrator/
+
+ENV PYTHONUNBUFFERED=1
+# ENTRYPOINT ["sh", "start.sh"]
+CMD ["/usr/bin/supervisord"]

config/Dockerfile.router

@@ -0,0 +1,12 @@
+FROM python:3.8.10
+
+COPY manager/requirements.txt /manager/
+
+RUN pip install -r /manager/requirements.txt
+
+COPY router/ /router/
+
+WORKDIR /router
+
+ENV PYTHONUNBUFFERED=1
+ENTRYPOINT ["sh", "start.sh"]

docker-compose.yaml

@@ -0,0 +1,105 @@
+services:
+  manager-celery:
+    build:
+      context: .
+      dockerfile: config/Dockerfile.manager
+    volumes:
+      - ./manager:/var/www/director/manager:z
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./shared/directorutil:/var/www/director/manager/directorutil:z
+      - ./shared/director-shell-keys/:/manager/director-shell-keys/:z
+    environment:
+      - CELERY=1
+    depends_on:
+      - redis
+      - registry
+      - manager-postgres
+      - postgres
+      - mysql
+  manager-web:
+    build:
+      context: .
+      dockerfile: config/Dockerfile.manager
+    ports:
+      - 8080:8080
+    volumes:
+      - ./manager:/var/www/director/manager:z
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./shared/directorutil:/var/www/director/manager/directorutil:z
+      - ./shared/director-shell-keys/:/manager/director-shell-keys/:z
+    depends_on:
+      - redis
+      - registry
+      - manager-postgres
+      - postgres
+      - mysql
+  orchestrator:
+    build:
+      context: .
+      dockerfile: config/Dockerfile.orchestrator
+    ports:
+      - "5000:5000"
+      - "5010:5010"
+    volumes:
+      - ./orchestrator:/var/www/director/orchestrator:z
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./shared/directorutil:/var/www/director/orchestrator/directorutil:z
+      - ./shared/director-shell-keys/:/etc/director-shell-keys/
+    depends_on:
+      - redis
+      - registry
+      - postgres
+      - mysql
+  router:
+    build:
+      context: .
+      dockerfile: config/Dockerfile.manager
+    volumes:
+      - ./manager:/manager:z
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./shared/directorutil:/router/directorutil:z
+    depends_on:
+      - redis
+      - registry
+      - postgres
+      - mysql
+  redis:
+    image: redis:alpine
+    ports:
+      - "6379"
+  registry:
+    image: registry:2
+    ports:
+      - "5002:5000"
+    volumes:
+      - ./registry-data:/var/lib/registry
+  manager-postgres:
+    image: postgres:9.6-alpine
+    ports:
+      - "5432"
+    volumes:
+      - ./postgres-manager-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: manager
+      POSTGRES_PASSWORD: pwd
+      POSTGRES_DB: manager
+  postgres:
+    image: postgres:9.6-alpine
+    ports:
+      - 5432:5432
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: director
+      POSTGRES_PASSWORD: director
+      POSTGRES_DB: director-postgres
+  mysql:
+    image: mysql:5.7
+    ports:
+      - 3306:3306
+    volumes:
+      - ./mysql-data:/var/lib/mysql
+    environment:
+      MYSQL_USER: director
+      MYSQL_PASSWORD: director
+      MYSQL_DATABASE: director-mysql

manager/director/settings/secret_dev.py

@@ -4,7 +4,7 @@
 
 DEBUG = True
 
-REDIS_HOST = "127.0.0.1"
+REDIS_HOST = "redis"
 REDIS_PORT = "6379"
 
 CHANNELS_REDIS_DB = 0
@@ -25,7 +25,7 @@
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.postgresql",
-        "HOST": "127.0.0.1",
+        "HOST": "manager-postgres",
         "PORT": "5432",
         "NAME": "manager",
         "USER": "manager",
@@ -40,9 +40,9 @@
     DATABASES["default"]["ENGINE"] = "django.db.backends.sqlite3"
     DATABASES["default"]["NAME"] = ":memory:"
 
-DIRECTOR_APPSERVER_HOSTS = ["localhost:5000"]
+DIRECTOR_APPSERVER_HOSTS = ["orchestrator:5000"]
 
-DIRECTOR_APPSERVER_WS_HOSTS = ["localhost:5010"]
+DIRECTOR_APPSERVER_WS_HOSTS = ["orchestrator:5010"]
 
 SITE_URL_FORMATS: Dict[Optional[str], str] = {
     None: "http://127.0.0.1:8081/{}/",

manager/requirements.txt

@@ -0,0 +1,75 @@
+aioredis==1.3.1
+amqp==5.0.6
+asgiref==3.3.4
+async-timeout==3.0.1
+attrs==20.3.0
+autobahn==21.3.1
+automat==20.2.0
+backcall==0.2.0
+billiard==3.6.4.0
+bleach==3.3.0
+celery==5.0.5
+certifi==2020.12.5
+cffi==1.14.5
+channels==3.0.3
+channels-redis==3.2.0
+chardet==4.0.0
+click==7.1.2
+click-didyoumean==0.0.3
+click-plugins==1.1.1
+click-repl==0.1.6
+constantly==15.1.0
+cryptography==3.4.7
+daphne==3.0.1
+decorator==5.0.5
+defusedxml==0.7.1
+django==2.2.20
+django-extensions==3.1.2
+django-redis-cache==3.0.0
+hiredis==2.0.0
+hyperlink==21.0.0
+idna==2.10
+importlib-metadata==3.10.0
+incremental==21.3.0
+ipython==7.22.0
+ipython-genutils==0.2.0
+jedi==0.18.0
+kombu==5.0.2
+markdown==3.3.4
+msgpack==1.0.2
+oauthlib==3.1.0
+packaging==20.9
+parso==0.8.2
+pexpect==4.8.0
+pickleshare==0.7.5
+prompt-toolkit==3.0.18
+psycopg2==2.8.6
+ptyprocess==0.7.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodome==3.10.1
+pygments==2.8.1
+pyjwt==2.0.1
+pyopenssl==20.0.1
+pyparsing==2.4.7
+python3-openid==3.2.0
+pytz==2021.1
+redis==3.5.3
+requests==2.25.1
+requests-oauthlib==1.3.0
+service-identity==18.1.0
+six==1.15.0
+social-auth-app-django==4.0.0
+social-auth-core==4.1.0
+sqlparse==0.4.1
+traitlets==5.0.5
+twisted==21.2.0
+txaio==21.2.1
+urllib3==1.26.4
+vine==5.0.0
+wcwidth==0.2.5
+webencodings==0.5.1
+websockets==8.1
+zipp==3.4.1
+zope.interface==5.3.0

manager/scripts/update-db.py

@@ -0,0 +1,37 @@
+from director.apps.sites.models import DatabaseHost
+
+remove_databases = [
+    {'hostname': 'postgres', 'port': 5432},
+    {'hostname': 'mysql', 'port': 3306},
+]
+for data in remove_databases:
+    DatabaseHost.objects.filter(**data).delete()
+
+databases = [
+    {
+        'hostname': 'postgres',
+        'port': 5432,
+        'dbms': 'postgres',
+        'admin_hostname': 'localhost',
+        'admin_port': 5433,
+        'admin_username': 'postgres',
+        'admin_password': 'pwd',
+    },
+    {
+        'hostname': 'mysql',
+        'port': 3306,
+        'dbms': 'mysql',
+        'admin_hostname': '127.0.0.1',
+        'admin_port': 3307,
+        'admin_username': 'root',
+        'admin_password': 'pwd',
+    },
+]
+
+for data in databases:
+    q = DatabaseHost.objects.filter(hostname=data['hostname'], port=data['port'])
+    if q.exists():
+        assert q.count() == 1, 'Please delete duplicate DatabaseHosts'
+        q.update(**data)
+    else:
+        DatabaseHost.objects.create(**data)

manager/start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+cp -r /etc/director-shell-keys/* /manager/director-shell-keys/
+
+if [ $CELERY ]; then
+    exec celery -A director worker --pool solo
+else
+    python3 manage.py migrate
+    python3 manage.py shell < scripts/update-db.py
+    exec python3 manage.py runserver 0.0.0.0:8080
+fi

orchestrator/requirements.txt

@@ -0,0 +1,19 @@
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+docker==4.4.4
+flask==1.1.2
+gunicorn==20.1.0
+idna==2.10
+itsdangerous==1.1.0
+jinja2==2.11.3
+markupsafe==1.1.1
+mysqlclient==2.0.3
+psycopg2==2.8.6
+pycryptodome==3.10.1
+requests==2.25.1
+six==1.15.0
+urllib3==1.26.4
+websocket-client==0.58.0
+websockets==8.1
+werkzeug==1.0.1

orchestrator/supervisord.conf

@@ -0,0 +1,17 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+
+[program:server]
+environment=FLASK_APP=orchestrator.app,FLASK_ENV=development
+command=flask run --host 0.0.0.0
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+
+[program:wsserver]
+command=python -m orchestrator.ws -b 0.0.0.0
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true