From f47f752f50423516b410547ec01f22aed2527f2e Mon Sep 17 00:00:00 2001 From: Francesco Bartoli Date: Sun, 30 Jun 2019 22:21:39 +0200 Subject: [PATCH] Add dev mode with regular Docker (#4514) * add env files for dev * Add docker-compose scripts for dev mode * Fix reference links to docs * Align with template link * Avoid clash between django and celery commands * Fix broken links * Align geoserver version in compose file for devel * Remove no longer used variable * Set env_file from root environment variable SET_DOCKER_ENV * Align prod and devel env files * Align filename extension of docker-compose file * Improve cmd logic for development * Use a single file to override for development mode * Add docs for developing with docker * Install ipdb for development --- .env | 1 + docker-compose-geoserver-server.yml | 2 +- ...ose.async.yaml => docker-compose.async.yml | 8 +-- docker-compose.development.yml | 68 +++++++++++++++++++ docker-compose.override.localhost.yml | 1 - docker-compose.override.yml | 1 - docker-compose.yml | 6 +- docs/devel/docker/index.rst | 58 ++++++++++++++++ docs/devel/index.rst | 7 +- entrypoint.sh | 27 +++++++- geonode/geoserver/helpers.py | 2 +- geonode/locale/ar/LC_MESSAGES/django.po | 4 +- geonode/locale/de/LC_MESSAGES/django.po | 4 +- geonode/locale/el/LC_MESSAGES/django.po | 2 +- geonode/locale/en/LC_MESSAGES/django.po | 2 +- geonode/locale/it/LC_MESSAGES/django.po | 2 +- geonode/tests/integration.py | 3 +- requirements_dev.txt | 1 + scripts/docker/env/development/celery.env | 43 ++++++++++++ scripts/docker/env/development/django.env | 46 ++++++++++--- scripts/docker/env/development/geoserver.env | 7 +- tasks.py | 6 ++ 22 files changed, 269 insertions(+), 32 deletions(-) rename docker-compose.async.yaml => docker-compose.async.yml (93%) create mode 100644 docker-compose.development.yml create mode 100644 docs/devel/docker/index.rst create mode 100644 requirements_dev.txt create mode 100644 scripts/docker/env/development/celery.env diff --git a/.env b/.env index 8ccf88bc06e..f12496006f6 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ COMPOSE_PROJECT_NAME=geonode +SET_DOCKER_ENV=production diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml index f34ffe6857a..5f5d7382cc8 100644 --- a/docker-compose-geoserver-server.yml +++ b/docker-compose-geoserver-server.yml @@ -26,7 +26,7 @@ services: volumes: - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/geoserver.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env ports: - "${GEOSERVER_SERVER_PORT}:8080" network_mode: "bridge" diff --git a/docker-compose.async.yaml b/docker-compose.async.yml similarity index 93% rename from docker-compose.async.yaml rename to docker-compose.async.yml index 2d79c019aa9..bc50584d58d 100644 --- a/docker-compose.async.yaml +++ b/docker-compose.async.yml @@ -14,7 +14,7 @@ services: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups env_file: - - ./scripts/docker/env/production/db.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/db.env elasticsearch: image: elasticsearch:2.4.1 @@ -56,7 +56,7 @@ services: - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/celery.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/celery.env geoserver: image: geonode/geoserver:2.14.3 @@ -75,7 +75,7 @@ services: volumes: - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/geoserver.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env django: restart: unless-stopped @@ -97,7 +97,7 @@ services: - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/django.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/django.env geonode: image: geonode/nginx:geoserver diff --git a/docker-compose.development.yml b/docker-compose.development.yml new file mode 100644 index 00000000000..d8f4c51d44e --- /dev/null +++ b/docker-compose.development.yml @@ -0,0 +1,68 @@ +version: '2.2' +services: + + celery: + build: . + command: celery worker --app=geonode.celery_app:app --broker=amqp://guest:guest@rabbitmq:5672/ -B -l DEBUG + depends_on: + - db + - elasticsearch + - rabbitmq + volumes: + - .:/usr/src/app + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + environment: + - DOCKER_ENV=${SET_DOCKER_ENV} + - IS_CELERY=True + - DEBUG=True + - GEONODE_LB_HOST_IP=localhost + - GEONODE_LB_PORT=80 + - SITEURL=http://localhost/ + - ALLOWED_HOSTS=['localhost', ] + - GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ + - GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ + + django: + build: . + command: python manage.py runserver --settings=geonode.settings 0.0.0.0:8000 + volumes: + - .:/usr/src/app + - statics:/mnt/volumes/statics + - geoserver-data-dir:/geoserver_data/data + environment: + - DOCKER_ENV=${SET_DOCKER_ENV} + - IS_CELERY=False + - DEBUG=True + - GEONODE_LB_HOST_IP=localhost + - GEONODE_LB_PORT=80 + - SITEURL=http://localhost/ + - ALLOWED_HOSTS=['localhost', ] + - GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ + - GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ + + geoserver: + depends_on: + - db + - elasticsearch + - rabbitmq + - data-dir-conf + environment: + - GEONODE_LB_HOST_IP=localhost + - GEONODE_LB_PORT=80 + + geonode: + image: geonode/nginx:${SET_DOCKER_ENV} + restart: unless-stopped + container_name: nginx4${COMPOSE_PROJECT_NAME} + stdin_open: true + # tty: true + labels: + org.geonode.component: nginx + org.geonode.instance.name: geonode + depends_on: + - django + - celery + - geoserver + ports: + - 80:80 diff --git a/docker-compose.override.localhost.yml b/docker-compose.override.localhost.yml index 9cecdec3b2e..928655c4512 100644 --- a/docker-compose.override.localhost.yml +++ b/docker-compose.override.localhost.yml @@ -21,4 +21,3 @@ services: environment: - GEONODE_LB_HOST_IP=localhost - GEONODE_LB_PORT=80 - # - NGINX_BASE_URL= diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 9cecdec3b2e..928655c4512 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -21,4 +21,3 @@ services: environment: - GEONODE_LB_HOST_IP=localhost - GEONODE_LB_PORT=80 - # - NGINX_BASE_URL= diff --git a/docker-compose.yml b/docker-compose.yml index 5dbd5a4088e..bec776b5faa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups env_file: - - ./scripts/docker/env/production/db.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/db.env geoserver: image: geonode/geoserver:2.14.3 @@ -31,7 +31,7 @@ services: volumes: - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/geoserver.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env django: restart: unless-stopped @@ -51,7 +51,7 @@ services: - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data env_file: - - ./scripts/docker/env/production/django.env + - ./scripts/docker/env/${SET_DOCKER_ENV}/django.env geonode: image: geonode/nginx:geoserver diff --git a/docs/devel/docker/index.rst b/docs/devel/docker/index.rst new file mode 100644 index 00000000000..19e00481c87 --- /dev/null +++ b/docs/devel/docker/index.rst @@ -0,0 +1,58 @@ +Start to develop with Docker +---------------------------- + +How to run the instance for development +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Set the variable SET_DOCKER_ENV for development +............................................... + +.. code-block:: shell + + vi .env + +Change to + +.. code-block:: shell + + SET_DOCKER_ENV=development + +Use dedicated docker-compose files while developing +................................................... + +.. note:: In this example we are going to keep ``localhost`` as the target IP for GeoNode + +.. code-block:: shell + + docker-compose -f docker-compose.async.yml -f docker-compose.development.yml up + +How to debug +............ + +.. note:: We are supposing to use ``ipdb`` for debugging which is already available as package from the container + +Stop the container for the ``django`` service: + +.. code-block:: shell + + docker-compose stop django + +Run the container again with the option for *service ports*: + +.. code-block:: shell + + docker-compose run \ + -e DOCKER_ENV=development \ + -e IS_CELERY=False \ + -e DEBUG=True \ + -e GEONODE_LB_HOST_IP=localhost \ + -e GEONODE_LB_PORT=80 \ + -e SITEURL=http://localhost/ \ + -e ALLOWED_HOSTS="['localhost', ]" \ + -e GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ \ + -e GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ \ + --rm --service-ports django python manage.py runserver --settings=geonode.settings 0.0.0.0:8000 + +Access the site on http://localhost/ + +.. note:: If you set an ``ipdb`` debug point with import ``ipdb ; ipdb.set_trace()`` then you should be facing its console and you can see the django server which is restarting at any change of your code from your local machine. \ No newline at end of file diff --git a/docs/devel/index.rst b/docs/devel/index.rst index 7bd3b6ea1d4..01e079ab6fc 100644 --- a/docs/devel/index.rst +++ b/docs/devel/index.rst @@ -1,2 +1,7 @@ How to Develop -============== \ No newline at end of file +============== + +.. toctree:: + :maxdepth: 3 + + docker/index \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 442fddec102..5ef88baf5be 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,7 +25,28 @@ echo DOCKER_ENV=$DOCKER_ENV if [ -z ${DOCKER_ENV} ] || [ ${DOCKER_ENV} = "development" ] then - echo "Executing standard Django server $cmd for Development" + /usr/local/bin/invoke prepare + echo "prepare task done" + /usr/local/bin/invoke fixtures + echo "fixture task done" + + if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ] + then + + cmd=$cmd + echo "Executing Celery server $cmd for Development" + + else + + echo "install requirements for development" + /usr/local/bin/invoke devrequirements + echo "refresh static data" + /usr/local/bin/invoke statics + echo "static data refreshed" + cmd=$cmd + echo "Executing standard Django server $cmd for Development" + + fi else @@ -38,11 +59,14 @@ else else if [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then + /usr/local/bin/invoke prepare echo "prepare task done" /usr/local/bin/invoke fixtures echo "fixture task done" + fi + /usr/local/bin/invoke initialized echo "initialized" @@ -57,4 +81,5 @@ else fi +echo "command to be executed is $cmd" exec $cmd diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py index 5e4d4852945..29f1aca16ec 100755 --- a/geonode/geoserver/helpers.py +++ b/geonode/geoserver/helpers.py @@ -73,7 +73,7 @@ msg = ( 'Please configure OGC_SERVER when enabling geonode.geoserver.' ' More info can be found at ' - 'http://docs.geonode.org/en/master/reference/developers/settings.html#ogc-server') + 'http://docs.geonode.org/en/master/basic/settings/index.html#ogc-server') raise ImproperlyConfigured(msg) diff --git a/geonode/locale/ar/LC_MESSAGES/django.po b/geonode/locale/ar/LC_MESSAGES/django.po index 67b2389d201..1a35b949e06 100644 --- a/geonode/locale/ar/LC_MESSAGES/django.po +++ b/geonode/locale/ar/LC_MESSAGES/django.po @@ -6266,8 +6266,8 @@ msgid "GeoNode is an open source platform for sharing geospatial data and maps." msgstr "GeoNode هو منصة مفتوحة المصدر لتبادل البيانات الجغرافية والخرائط." #: geonode/templates/index.html:23 -msgid "http://docs.geonode.org/en/master/tutorials/users/index.html" -msgstr "http://docs.geonode.org/en/master/tutorials/users/index.html" +msgid "http://docs.geonode.org/en/master/usage" +msgstr "http://docs.geonode.org/en/master/usage" #: geonode/templates/index.html:23 msgid "Get Started »" diff --git a/geonode/locale/de/LC_MESSAGES/django.po b/geonode/locale/de/LC_MESSAGES/django.po index 3be0681c2ba..52d3c7cc3a8 100644 --- a/geonode/locale/de/LC_MESSAGES/django.po +++ b/geonode/locale/de/LC_MESSAGES/django.po @@ -1084,8 +1084,8 @@ msgstr "GeoNode ist eine Open-Source-Plattform zur Veröffentlichung von Geodate #: geonode/contrib/geosites/site_template/templates/site_index.html:14 #: geonode/contrib/geosites/templates/master_index.html:15 geonode/templates/index.html:23 -msgid "http://docs.geonode.org/en/master/tutorials/users/index.html" -msgstr "http://docs.geonode.org/en/master/tutorials/users/index.html" +msgid "http://docs.geonode.org/en/master/usage" +msgstr "http://docs.geonode.org/en/master/usage" #: geonode/contrib/geosites/site_template/templates/site_index.html:15 #: geonode/contrib/geosites/templates/master_index.html:16 geonode/templates/index.html:23 diff --git a/geonode/locale/el/LC_MESSAGES/django.po b/geonode/locale/el/LC_MESSAGES/django.po index b4a7d39ea4c..ea7fcdcad74 100644 --- a/geonode/locale/el/LC_MESSAGES/django.po +++ b/geonode/locale/el/LC_MESSAGES/django.po @@ -1182,7 +1182,7 @@ msgstr "" #: geonode/contrib/geosites/site_template/templates/site_index.html:14 #: geonode/contrib/geosites/templates/master_index.html:15 #: geonode/templates/index.html:23 -msgid "http://docs.geonode.org/en/master/tutorials/users/index.html" +msgid "http://docs.geonode.org/en/master/usage" msgstr "" #: geonode/contrib/geosites/site_template/templates/site_index.html:15 diff --git a/geonode/locale/en/LC_MESSAGES/django.po b/geonode/locale/en/LC_MESSAGES/django.po index 612a4dd4b3c..22919b2340d 100644 --- a/geonode/locale/en/LC_MESSAGES/django.po +++ b/geonode/locale/en/LC_MESSAGES/django.po @@ -7827,7 +7827,7 @@ msgid "home" msgstr "" #: geonode/templates/index.html:23 -msgid "http://docs.geonode.org/en/master/tutorials/users/index.html" +msgid "http://docs.geonode.org/en/master/usage" msgstr "" #: geonode/templates/index.html:23 diff --git a/geonode/locale/it/LC_MESSAGES/django.po b/geonode/locale/it/LC_MESSAGES/django.po index ce7716c80d6..cf7d577355d 100644 --- a/geonode/locale/it/LC_MESSAGES/django.po +++ b/geonode/locale/it/LC_MESSAGES/django.po @@ -7459,7 +7459,7 @@ msgid "home" msgstr "" #: geonode/templates/index.html:23 -msgid "http://docs.geonode.org/en/master/tutorials/users/index.html" +msgid "http://docs.geonode.org/en/master/usage" msgstr "" #: geonode/templates/index.html:23 diff --git a/geonode/tests/integration.py b/geonode/tests/integration.py index 409e6d8abd0..9a1a78a1fb6 100644 --- a/geonode/tests/integration.py +++ b/geonode/tests/integration.py @@ -104,10 +104,9 @@ def zip_dir(basedir, archivename): r""" HOW TO RUN THE TESTS -------------------- - (https://github.com/GeoNode/geonode/blob/master/docs/tutorials/devel/testing.txt) 1) - (https://github.com/GeoNode/geonode/blob/master/docs/tutorials/devel/envsetup/paver.txt) + (http://docs.geonode.org/en/master/install/core/index.html?highlight=paver#run-geonode-for-the-first-time-in-debug-mode) $ paver setup diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000000..0f927da3275 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1 @@ +ipdb \ No newline at end of file diff --git a/scripts/docker/env/development/celery.env b/scripts/docker/env/development/celery.env new file mode 100644 index 00000000000..7279c27ee84 --- /dev/null +++ b/scripts/docker/env/development/celery.env @@ -0,0 +1,43 @@ +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode +GEONODE_LB_HOST_IP +GEONODE_LB_PORT +DEFAULT_BACKEND_DATASTORE=datastore +GEONODE_DATABASE=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +ASYNC_SIGNALS=True +BROKER_URL=amqp://guest:guest@rabbitmq:5672 +DOCKER_ENV=development +IS_CELERY=True +DEBUG=True +C_FORCE_ROOT=1 +SITEURL=http://localhost/ +# replaced with defaults in settings +GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ +GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +OGC_REQUEST_TIMEOUT=300 +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db +ALLOWED_HOSTS=['django', '*'] +ADMIN_EMAILS +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False +GEOGIG_ENABLED=False +HAYSTACK_SEARCH=False +HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ +HAYSTACK_ENGINE_INDEX_NAME=haystack +HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 + + +ALLOWED_DOCUMENT_TYPES=['doc', 'docx', 'gif', 'jpg', 'jpeg', 'ods', 'odt', 'odp', 'pdf', 'png','ppt', 'pptx', 'rar', 'sld', 'tif', 'tiff', 'txt', 'xls', 'xlsx', 'xml', 'zip', 'gz', 'qml'] +MAX_DOCUMENT_SIZE=2 + +# GEOSERVER_ADMIN_PASSWORD=admin +# See https://github.com/geosolutions-it/geonode-generic/issues/28 +# to see why we force API version to 1.24 +DOCKER_API_VERSION="1.24" diff --git a/scripts/docker/env/development/django.env b/scripts/docker/env/development/django.env index da3ea06c91d..66c7f17a827 100644 --- a/scripts/docker/env/development/django.env +++ b/scripts/docker/env/development/django.env @@ -1,14 +1,44 @@ -GEONODE_INSTANCE_NAME=geonode -BROKER_URL=amqp://guest:guest@rabbitmq:5672/ DJANGO_SETTINGS_MODULE=geonode.settings -GEOSERVER_INTERNAL_URL=http://geoservices:8080/geoserver/ -ALLOWED_HOSTS=['django',] +GEONODE_INSTANCE_NAME=geonode +GEONODE_LB_HOST_IP +GEONODE_LB_PORT +DEFAULT_BACKEND_DATASTORE=datastore +GEONODE_DATABASE=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_PASSWORD=geonode_data +ASYNC_SIGNALS=True +BROKER_URL=amqp://guest:guest@rabbitmq:5672 DOCKER_ENV=development -DOCKER_HOST -PUBLIC_PORT=8000 -DOCKER_HOST_IP +IS_CELERY=False DEBUG=True C_FORCE_ROOT=1 +SITEURL=http://localhost/ +# replaced with defaults in settings +GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ +GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ +GEOSERVER_LOCATION=http://geoserver:8080/geoserver/ +OGC_REQUEST_TIMEOUT=300 +STATIC_ROOT=/mnt/volumes/statics/static/ +MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +GEOIP_PATH=/mnt/volumes/statics/geoip.db +ALLOWED_HOSTS=['django', '*'] +ADMIN_EMAILS +DEFAULT_BACKEND_UPLOADER=geonode.importer +TIME_ENABLED=True +MOSAIC_ENABLED=False +HAYSTACK_SEARCH=False +HAYSTACK_ENGINE_URL=http://elasticsearch:9200/ +HAYSTACK_ENGINE_INDEX_NAME=haystack +HAYSTACK_SEARCH_RESULTS_PER_PAGE=200 + + +ALLOWED_DOCUMENT_TYPES=['doc', 'docx', 'gif', 'jpg', 'jpeg', 'ods', 'odt', 'odp', 'pdf', 'png','ppt', 'pptx', 'rar', 'sld', 'tif', 'tiff', 'txt', 'xls', 'xlsx', 'xml', 'zip', 'gz', 'qml'] +MAX_DOCUMENT_SIZE=2 + + + +# GEOSERVER_ADMIN_PASSWORD=admin # See https://github.com/geosolutions-it/geonode-generic/issues/28 # to see why we force API version to 1.24 -DOCKER_API_VERSION: "1.24" +DOCKER_API_VERSION="1.24" diff --git a/scripts/docker/env/development/geoserver.env b/scripts/docker/env/development/geoserver.env index a2569028eac..857a592e1db 100644 --- a/scripts/docker/env/development/geoserver.env +++ b/scripts/docker/env/development/geoserver.env @@ -1,4 +1,7 @@ -DOCKER_HOST -PUBLIC_PORT=8000 +DOCKERHOST DOCKER_HOST_IP +GEONODE_LB_HOST_IP +GEONODE_LB_PORT +PUBLIC_PORT=80 +NGINX_BASE_URL GEOSERVER_JAVA_OPTS=-Djava.awt.headless=true -XX:MaxPermSize=512m -XX:PermSize=256m -Xms512m -Xmx2048m -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Duser.timezone=GMT -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=true diff --git a/tasks.py b/tasks.py index aef121844d4..e697f582d03 100755 --- a/tasks.py +++ b/tasks.py @@ -115,6 +115,12 @@ def initialized(ctx): ctx.run('date > /mnt/volumes/statics/geonode_init.lock') +@task +def devrequirements(ctx): + print "*********************install dev requirements**********************" + ctx.run('pip install -r requirements_dev.txt --upgrade') + + def _update_db_connstring(): user = os.getenv('GEONODE_DATABASE', 'geonode') pwd = os.getenv('GEONODE_DATABASE_PASSWORD', 'geonode')