diff --git a/workspace/.env b/workspace/.env index 942760d8c6..557c58e938 100644 --- a/workspace/.env +++ b/workspace/.env @@ -5,6 +5,8 @@ #git update-index --no-skip-worktree workspace/.env COMPOSE_PROJECT_NAME=stack-engage +STARTUP_CMD="/venv/bin/uwsgi --http-auto-chunked --http-keepalive --thunder-lock /rapidpro/uwsgi_conf.ini" + IMG_TAG_STACK=ci-develop IMG_TAG_ENGAGE=${IMG_TAG_STACK} IMG_TAG_COURIER=${IMG_TAG_STACK} @@ -33,6 +35,8 @@ PG_DB_PSWD=postgres RP_DB_URL=postgres://${PG_DB_USER}:${PG_DB_PSWD}@postgresql:5432/rapidpro?sslmode=disable PO_DB_URL=postgres://${PG_DB_USER}:${PG_DB_PSWD}@postgresql:5432/postoffice?sslmode=disable +POSTOFFICE_ENDPOINT=http://host.docker.internal/postoffice/ + # generated values that may need to be replaced now and then (especially when starting from scratch) PO_RAPIDPRO_APIKEY=REPLACE_ME_WITH_OUTPUT_FROM_engage_initialsetup.py POST_OFFICE_API_KEY=REPLACE_ME diff --git a/workspace/docker-compose.yml b/workspace/docker-compose.yml index 8a13fb7686..be19eabd6a 100644 --- a/workspace/docker-compose.yml +++ b/workspace/docker-compose.yml @@ -11,27 +11,19 @@ services: ports: - 8000:8000 volumes: - - data4media:/rapidpro/media #---- <*)))><: I found placing the local-static files far outside #---- the IDE/git pervuew prevented Docker from locking up. - - ~/engage-sitestatic:/rapidpro/sitestatic #- ./local_settings.py:/rapidpro/temba/local_settings.py + - ./uwsgi_conf.ini:/rapidpro/uwsgi_conf.ini - ./engage_initialsetup.py:/rapidpro/engage_initialsetup.py - - ../scm/src-refresh.sh:/opt/dev/src-refresh.sh - - ../docker/customizations/any:/opt/dev/any - - ../docker/customizations/engage:/opt/dev/brand - #- ../docker/customizations/generic:/opt/dev/brand - - ../engage:/opt/dev/rp/engage - - ../temba:/opt/dev/rp/temba + environment: + - STARTUP_CMD=${STARTUP_CMD} #- SUB_DIR=UNSUPPORTED-since-4.0.12 - SEND_CALLS=on - SEND_EMAILS=off - SEND_MESSAGES=on - SEND_WEBHOOKS=on - - CLEAR_COMPRESSOR_CACHE=off - - MANAGEPY_COLLECTSTATIC=off - - MANAGEPY_COMPRESS=off - MANAGEPY_INIT_DB=off - MANAGEPY_MIGRATE=off - DOMAIN_NAME=localhost @@ -76,24 +68,17 @@ services: # copy & paste the result in .env between the single quotes. #OIDC_RSA_PRIVATE_KEY_BASE64=' #' - - OIDC_RSA_PRIVATE_KEY_BASE64 - #- KEYCLOAK_URL=http://host.docker.internal:8008 - - KEYCLOAK_CLIENT_ID - - KEYCLOAK_CLIENT_SECRET - VERSION_TAG=ci-local-dev - VERSION_CI=1.0.0 #---- <*)))><: I found these settings worked best for dev - UWSGI_DISABLE_LOGGING=True - - DJANGO_DEBUG=on - - DJANGO_COMPRESSOR=off + - DJANGO_DEBUG=off - DJANGO_LOG_LEVEL=WARNING - LOG_LEVEL=DEBUG - - DEV_STATIC=on - - DEV_CODE_ONLY=0 - - CLEAR_SITESTATIC=0 + - IS_PROD=on - celery: - container_name: celery + celery_worker: + container_name: celery_worker image: istresearch/p4-engage:$IMG_TAG_ENGAGE depends_on: - engage @@ -113,7 +98,31 @@ services: - SECRET_KEY=notsecureatall - POST_OFFICE_API_KEY - POST_OFFICE_API_URL=http://host.docker.internal:8088/postoffice - command: ["/venv/bin/celery", "--beat", "--app=temba", "worker", "--loglevel=INFO", "--queues=celery,flows,msgs,handler"] + command: ["/venv/bin/celery", "worker", "--loglevel=INFO", "--queues=celery,flows,msgs,handler"] + restart: unless-stopped + + celery_beat: + container_name: celery_beat + image: istresearch/p4-engage:$IMG_TAG_ENGAGE + depends_on: + - engage + links: + - redis + - postgresql + volumes: + - data4media:/rapidpro/media + environment: + - SEND_CALLS=on + - SEND_MESSAGES=on + - SEND_AIRTIME=on + - SEND_WEBHOOKS=on + - ELASTICSEARCH_URL=http://elasticsearch:9200 + - DATABASE_URL=${RP_DB_URL} + - REDIS_URL=redis://redis:6379/1 + - SECRET_KEY=notsecureatall + - POST_OFFICE_API_KEY + - POST_OFFICE_API_URL=http://host.docker.internal:8088/postoffice + command: [ "/venv/bin/celery", "--app=temba", "beat", "--loglevel=INFO" ] restart: unless-stopped courier: @@ -227,11 +236,11 @@ services: nginx: container_name: nginx - image: nginx + image: istresearch/p4-engage:nginx-$IMG_TAG_ENGAGE volumes: - - ./nginx.conf:/etc/nginx/nginx.conf + - ./nginx-server-locations.conf:/etc/nginx/in_server.d/99-loc-website.conf ports: - - 80:8888 + - "80:8001" depends_on: - engage restart: unless-stopped @@ -290,17 +299,19 @@ services: postgresql: container_name: postgresql - image: mdillon/postgis:10 + image: kartoza/postgis:14-3.3 environment: - - POSTGRES_DB=rapidpro + - POSTGRES_DBNAME=rapidpro,postoffice - POSTGRES_USER=${PG_DB_USER} - - POSTGRES_PASSWORD=${PG_DB_PSWD} + - POSTGRES_PASS=${PG_DB_PSWD} + - POSTGRES_MULTIPLE_EXTENSIONS=postgis,hstore,postgis_topology,postgis_raster,uuid-ossp + - ALL_DATABASES=TRUE volumes: - - ./postgres-rp-extensions.sh:/docker-entrypoint-initdb.d/pg01.sh + #- ./postgres-rp-extensions.sh:/docker-entrypoint-initdb.d/pg01.sh # NOTE: If you have an existing rp db, run this command to create PO schema via: # docker exec -it postgresql docker-entrypoint-initdb.d/pg02.sh - - ./postgres-po-init.sh:/docker-entrypoint-initdb.d/pg02.sh - - data4pgsql:/var/lib/postgresql/data + #- ./postgres-po-init.sh:/docker-entrypoint-initdb.d/pg02.sh + - data4pgsql:/var/lib/postgresql ports: - "5432:5432" #command: ["postgres", "-c", "log_statement=all"] @@ -322,17 +333,6 @@ services: - debug=true restart: unless-stopped - keycloak: - container_name: keycloak - image: quay.io/keycloak/keycloak:17.0.0 - entrypoint: /opt/keycloak/bin/kc.sh start-dev - ports: - - 8008:8080 - environment: - - KEYCLOAK_ADMIN=admin - - KEYCLOAK_ADMIN_PASSWORD=admin - restart: unless-stopped - volumes: data4pgsql: data4es: diff --git a/workspace/engage_initialsetup.py b/workspace/engage_initialsetup.py index e60afb2f07..4642aeea02 100644 --- a/workspace/engage_initialsetup.py +++ b/workspace/engage_initialsetup.py @@ -1,3 +1,5 @@ +user_fname = "dev" +user_lname = "one" user_email = "dev@twosixtech.com" user_pass = "abc123" superuser_name = "admin" @@ -17,7 +19,7 @@ from django.utils import timezone from temba.api.models import APIToken -from temba.orgs.models import Org, OrgRole +from temba.orgs.models import Org, OrgRole, User as TembaUser # Create Super User (named admin) superuser = User.objects.create_superuser(superuser_name, superuser_email, superuser_pass) @@ -34,7 +36,7 @@ org.initialize(topup_size=org_topup) #Create user and set as Org admin -user = Org.create_user(user_email, user_pass) +user = TembaUser.create(user_email, user_fname, user_lname, user_pass) org.add_user(user, OrgRole.ADMINISTRATOR) token = APIToken.get_or_create(org, superuser) diff --git a/workspace/nginx-server-locations.conf b/workspace/nginx-server-locations.conf new file mode 100644 index 0000000000..0b0482c503 --- /dev/null +++ b/workspace/nginx-server-locations.conf @@ -0,0 +1,37 @@ +set $docker_courier host.docker.internal:8080; +set $docker_mailroom host.docker.internal:8091; +set $docker_postoffice host.docker.internal:8088; +set $docker_engage host.docker.internal:8000; + +location ^~ /mr/ { + proxy_set_header Host $http_host; + proxy_pass http://$docker_mailroom; + break; +} + +location ~ /c/ { + proxy_set_header Host $http_host; + proxy_pass http://$docker_courier; + break; +} + +location ^~ /postoffice/ { + proxy_set_header Host $http_host; + proxy_pass http://$docker_postoffice; + break; +} + +location ^~ /graphql/ { + proxy_set_header Host $http_host; + proxy_pass http://$docker_postoffice; + break; +} + +# Finally, send all non-media requests to the Django server. +location / { + proxy_set_header Host $http_host; + proxy_pass http://$docker_engage; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + break; +} \ No newline at end of file diff --git a/workspace/nginx.conf b/workspace/nginx.conf deleted file mode 100644 index 6cb2c8472a..0000000000 --- a/workspace/nginx.conf +++ /dev/null @@ -1,65 +0,0 @@ -events { -} - -http { - resolver 127.0.0.11; - - # configuration of the server - server { - # the port your site will be served on - listen 8888; - # the domain name it will serve forproxy_set_header - server_name localhost; # substitute your machine's IP address or FQDN - charset utf-8; - # max upload size - client_max_body_size 75M; # adjust to taste - - include /etc/nginx/mime.types; - underscores_in_headers on; - - # All Courier URLs goto Courier - location ^~ /c/ { - proxy_set_header Host $http_host; - proxy_pass http://host.docker.internal:8080; - break; - } - - # all Mailroom URLs go to Mailroom - location ^~ /mr/ { - proxy_set_header Host $http_host; - proxy_pass http://host.docker.internal:8091; - break; - } - - # Postoffice URLs - location ^~ /postoffice/ { - proxy_set_header Host $http_host; - proxy_pass http://host.docker.internal:8088; - break; - } - - # GraphQL playground URLs - location ^~ /graphql/ { - proxy_set_header Host $http_host; - proxy_pass http://host.docker.internal:8088; - break; - } - - # Note: I never did get this working right - location ^~ /s3/ { - proxy_set_header Host $http_host; - #proxy_set_header Accept ""; - proxy_pass http://host.docker.internal:9090/; - break; - } - - # Finally, send all non-media requests to the Django server. - location / { - proxy_set_header Host $http_host; - proxy_pass http://host.docker.internal:8000; - proxy_read_timeout 600s; - proxy_send_timeout 600s; - break; - } - } -} \ No newline at end of file diff --git a/workspace/uwsgi_conf.ini b/workspace/uwsgi_conf.ini new file mode 100644 index 0000000000..57ec798849 --- /dev/null +++ b/workspace/uwsgi_conf.ini @@ -0,0 +1,41 @@ +[uwsgi] +strict = true +master = true +enable-threads = true +vacuum = true ; Delete sockets during shutdown +single-interpreter = true +die-on-term = true ; Shutdown when receiving SIGTERM (default is respawn) +need-app = true + +; @see https://stackoverflow.com/questions/73518167/uwsgi-segmentation-fault +py-autoreload = 30 ; monitor file changes (good for local dev), disable! + +;disable-logging = true ; Disable built-in logging +;log-4xx = true ; but log 4xx's anyway +;log-5xx = true ; and 5xx's + +harakiri = 120 ; forcefully kill workers after X seconds +;py-callos-afterfork = true ; allow workers to trap signals +buffer-size = 32768 + +; @see https://stackoverflow.com/questions/69396898/trying-to-figure-out-uwsgi-thread-workers-configuration +;max-requests = 1000 ; Restart workers after this many requests +max-worker-lifetime = 3600 ; Restart workers after this many seconds +max-worker-lifetime-delta = 100 ; Restart workers taking PID into account so not all at once +reload-on-rss = 1200 ; Restart workers after this much resident memory +worker-reload-mercy = 120 ; How long to wait before forcefully killing workers + +threads = 1 ; num threads allowed per worker +processes = 20 ; Maximum number of workers allowed; formula = ( (# of CPUs)*2 ) + 1 + +cheaper-algo = busyness +cheaper = 1 ; Minimum number of workers allowed +cheaper-initial = 5 ; Workers created at startup +cheaper-overload = 5 ; Length of a cycle in seconds +cheaper-step = 2 ; How many workers to spawn at a time + +cheaper-busyness-multiplier = 30 ; How many cycles to wait before killing workers +cheaper-busyness-min = 20 ; Below this threshold, kill workers (if stable for multiplier cycles) +cheaper-busyness-max = 70 ; Above this threshold, spawn new workers +cheaper-busyness-backlog-alert = 16 ; Spawn emergency workers if more than this many requests are waiting in the queue +cheaper-busyness-backlog-step = 2 ; How many emergency workers to create if there are too many requests in the queue