Add dev mode with regular Docker (#4514)

* add env files for dev * Add docker-compose scripts for dev mode * Fix reference links to docs * Align with template link * Avoid clash between django and celery commands * Fix broken links * Align geoserver version in compose file for devel * Remove no longer used variable * Set env_file from root environment variable SET_DOCKER_ENV * Align prod and devel env files * Align filename extension of docker-compose file * Improve cmd logic for development * Use a single file to override for development mode * Add docs for developing with docker * Install ipdb for development
GeoNode · Jun 30, 2019 · f47f752 · f47f752
1 parent 2073bee
commit f47f752
Show file tree

Hide file tree

Showing 22 changed files with 269 additions and 32 deletions.
diff --git a/.env b/.env
@@ -1 +1,2 @@
 COMPOSE_PROJECT_NAME=geonode
+SET_DOCKER_ENV=production
diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml
@@ -26,7 +26,7 @@ services:
     volumes:
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/geoserver.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env
     ports:
       - "${GEOSERVER_SERVER_PORT}:8080"
     network_mode: "bridge"

diff --git a/docker-compose.async.yaml → docker-compose.async.yml b/docker-compose.async.yaml → docker-compose.async.yml
@@ -14,7 +14,7 @@ services:
       - dbdata:/var/lib/postgresql/data
       - dbbackups:/pg_backups
     env_file:
-      - ./scripts/docker/env/production/db.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/db.env
 
   elasticsearch:
     image: elasticsearch:2.4.1
@@ -56,7 +56,7 @@ services:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/celery.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/celery.env
 
   geoserver:
     image: geonode/geoserver:2.14.3
@@ -75,7 +75,7 @@ services:
     volumes:
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/geoserver.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env
 
   django:
     restart: unless-stopped
@@ -97,7 +97,7 @@ services:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/django.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/django.env
 
   geonode:
     image: geonode/nginx:geoserver

diff --git a/docker-compose.development.yml b/docker-compose.development.yml
@@ -0,0 +1,68 @@
+version: '2.2'
+services:
+
+  celery:
+    build: .
+    command: celery worker --app=geonode.celery_app:app --broker=amqp://guest:guest@rabbitmq:5672/ -B -l DEBUG
+    depends_on:
+      - db
+      - elasticsearch
+      - rabbitmq
+    volumes:
+      - .:/usr/src/app
+      - statics:/mnt/volumes/statics
+      - geoserver-data-dir:/geoserver_data/data
+    environment:
+      - DOCKER_ENV=${SET_DOCKER_ENV}
+      - IS_CELERY=True
+      - DEBUG=True
+      - GEONODE_LB_HOST_IP=localhost
+      - GEONODE_LB_PORT=80
+      - SITEURL=http://localhost/
+      - ALLOWED_HOSTS=['localhost', ]
+      - GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/
+      - GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/
+
+  django:
+    build: .
+    command: python manage.py runserver --settings=geonode.settings 0.0.0.0:8000
+    volumes:
+      - .:/usr/src/app
+      - statics:/mnt/volumes/statics
+      - geoserver-data-dir:/geoserver_data/data
+    environment:
+      - DOCKER_ENV=${SET_DOCKER_ENV}
+      - IS_CELERY=False
+      - DEBUG=True
+      - GEONODE_LB_HOST_IP=localhost
+      - GEONODE_LB_PORT=80
+      - SITEURL=http://localhost/
+      - ALLOWED_HOSTS=['localhost', ]
+      - GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/
+      - GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/
+
+  geoserver:
+    depends_on:
+      - db
+      - elasticsearch
+      - rabbitmq
+      - data-dir-conf
+    environment:
+      - GEONODE_LB_HOST_IP=localhost
+      - GEONODE_LB_PORT=80
+
+  geonode:
+    image: geonode/nginx:${SET_DOCKER_ENV}
+    restart: unless-stopped
+    container_name: nginx4${COMPOSE_PROJECT_NAME}
+    stdin_open: true
+    # tty: true
+    labels:
+        org.geonode.component: nginx
+        org.geonode.instance.name: geonode
+    depends_on:
+      - django
+      - celery
+      - geoserver
+    ports:
+      - 80:80
diff --git a/docker-compose.override.localhost.yml b/docker-compose.override.localhost.yml
@@ -21,4 +21,3 @@ services:
     environment:
       - GEONODE_LB_HOST_IP=localhost
       - GEONODE_LB_PORT=80
-  #    - NGINX_BASE_URL=
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -21,4 +21,3 @@ services:
     environment:
       - GEONODE_LB_HOST_IP=localhost
       - GEONODE_LB_PORT=80
-  #    - NGINX_BASE_URL=
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,7 +14,7 @@ services:
       - dbdata:/var/lib/postgresql/data
       - dbbackups:/pg_backups
     env_file:
-      - ./scripts/docker/env/production/db.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/db.env
 
   geoserver:
     image: geonode/geoserver:2.14.3
@@ -31,7 +31,7 @@ services:
     volumes:
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/geoserver.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env
 
   django:
     restart: unless-stopped
@@ -51,7 +51,7 @@ services:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/production/django.env
+      - ./scripts/docker/env/${SET_DOCKER_ENV}/django.env
 
   geonode:
     image: geonode/nginx:geoserver

diff --git a/docs/devel/docker/index.rst b/docs/devel/docker/index.rst
@@ -0,0 +1,58 @@
+Start to develop with Docker
+----------------------------
+
+How to run the instance for development
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Set the variable SET_DOCKER_ENV for development
+...............................................
+
+.. code-block:: shell
+
+    vi .env
+
+Change to
+
+.. code-block:: shell
+
+    SET_DOCKER_ENV=development
+
+Use dedicated docker-compose files while developing
+...................................................
+
+.. note:: In this example we are going to keep ``localhost`` as the target IP for GeoNode
+
+.. code-block:: shell
+
+    docker-compose -f docker-compose.async.yml -f docker-compose.development.yml up
+
+How to debug
+............
+
+.. note:: We are supposing to use ``ipdb`` for debugging which is already available as package from the container
+
+Stop the container for the ``django`` service:
+
+.. code-block:: shell
+
+    docker-compose stop django
+
+Run the container again with the option for *service ports*:
+
+.. code-block:: shell
+
+    docker-compose run \
+        -e DOCKER_ENV=development \
+        -e IS_CELERY=False \
+        -e DEBUG=True \
+        -e GEONODE_LB_HOST_IP=localhost \
+        -e GEONODE_LB_PORT=80 \
+        -e SITEURL=http://localhost/ \
+        -e ALLOWED_HOSTS="['localhost', ]" \
+        -e GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/ \
+        -e GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/ \
+        --rm --service-ports django python manage.py runserver --settings=geonode.settings 0.0.0.0:8000
+
+Access the site on http://localhost/
+
+.. note:: If you set an ``ipdb`` debug point with import ``ipdb ; ipdb.set_trace()`` then you should be facing its console and you can see the django server which is restarting at any change of your code from your local machine.
diff --git a/docs/devel/index.rst b/docs/devel/index.rst
@@ -1,2 +1,7 @@
 How to Develop
-==============
+==============
+
+.. toctree::
+    :maxdepth: 3
+
+    docker/index
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -25,7 +25,28 @@ echo DOCKER_ENV=$DOCKER_ENV
 if [ -z ${DOCKER_ENV} ] || [ ${DOCKER_ENV} = "development" ]
 then
 
-    echo "Executing standard Django server $cmd for Development"
+    /usr/local/bin/invoke prepare
+    echo "prepare task done"
+    /usr/local/bin/invoke fixtures
+    echo "fixture task done"
+
+    if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ]
+    then
+
+        cmd=$cmd
+        echo "Executing Celery server $cmd for Development"
+
+    else
+
+        echo "install requirements for development"
+        /usr/local/bin/invoke devrequirements
+        echo "refresh static data"
+        /usr/local/bin/invoke statics
+        echo "static data refreshed"
+        cmd=$cmd
+        echo "Executing standard Django server $cmd for Development"
+
+    fi
 
 else
 
@@ -38,11 +59,14 @@ else
     else
 
         if [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then
+
             /usr/local/bin/invoke prepare
             echo "prepare task done"
             /usr/local/bin/invoke fixtures
             echo "fixture task done"
+
         fi
+
         /usr/local/bin/invoke initialized
         echo "initialized"
 
@@ -57,4 +81,5 @@ else
 
 fi
 
+echo "command to be executed is $cmd"
 exec $cmd
diff --git a/geonode/geoserver/helpers.py b/geonode/geoserver/helpers.py
@@ -73,7 +73,7 @@
     msg = (
         'Please configure OGC_SERVER when enabling geonode.geoserver.'
         ' More info can be found at '
-        'http://docs.geonode.org/en/master/reference/developers/settings.html#ogc-server')
+        'http://docs.geonode.org/en/master/basic/settings/index.html#ogc-server')
     raise ImproperlyConfigured(msg)
 
 

diff --git a/geonode/locale/ar/LC_MESSAGES/django.po b/geonode/locale/ar/LC_MESSAGES/django.po
@@ -6266,8 +6266,8 @@ msgid "GeoNode is an open source platform for sharing geospatial data and maps."
 msgstr "GeoNode هو منصة مفتوحة المصدر لتبادل البيانات الجغرافية والخرائط."
 
 #: geonode/templates/index.html:23
-msgid "http://docs.geonode.org/en/master/tutorials/users/index.html"
-msgstr "http://docs.geonode.org/en/master/tutorials/users/index.html"
+msgid "http://docs.geonode.org/en/master/usage"
+msgstr "http://docs.geonode.org/en/master/usage"
 
 #: geonode/templates/index.html:23
 msgid "Get Started &raquo;"

diff --git a/geonode/locale/de/LC_MESSAGES/django.po b/geonode/locale/de/LC_MESSAGES/django.po
@@ -1084,8 +1084,8 @@ msgstr "GeoNode ist eine Open-Source-Plattform zur Veröffentlichung von Geodate
 
 #: geonode/contrib/geosites/site_template/templates/site_index.html:14
 #: geonode/contrib/geosites/templates/master_index.html:15 geonode/templates/index.html:23
-msgid "http://docs.geonode.org/en/master/tutorials/users/index.html"
-msgstr "http://docs.geonode.org/en/master/tutorials/users/index.html"
+msgid "http://docs.geonode.org/en/master/usage"
+msgstr "http://docs.geonode.org/en/master/usage"
 
 #: geonode/contrib/geosites/site_template/templates/site_index.html:15
 #: geonode/contrib/geosites/templates/master_index.html:16 geonode/templates/index.html:23

diff --git a/geonode/locale/el/LC_MESSAGES/django.po b/geonode/locale/el/LC_MESSAGES/django.po
@@ -1182,7 +1182,7 @@ msgstr ""
 #: geonode/contrib/geosites/site_template/templates/site_index.html:14
 #: geonode/contrib/geosites/templates/master_index.html:15
 #: geonode/templates/index.html:23
-msgid "http://docs.geonode.org/en/master/tutorials/users/index.html"
+msgid "http://docs.geonode.org/en/master/usage"
 msgstr ""
 
 #: geonode/contrib/geosites/site_template/templates/site_index.html:15

diff --git a/geonode/locale/en/LC_MESSAGES/django.po b/geonode/locale/en/LC_MESSAGES/django.po
@@ -7827,7 +7827,7 @@ msgid "home"
 msgstr ""
 
 #: geonode/templates/index.html:23
-msgid "http://docs.geonode.org/en/master/tutorials/users/index.html"
+msgid "http://docs.geonode.org/en/master/usage"
 msgstr ""
 
 #: geonode/templates/index.html:23

diff --git a/geonode/locale/it/LC_MESSAGES/django.po b/geonode/locale/it/LC_MESSAGES/django.po
@@ -7459,7 +7459,7 @@ msgid "home"
 msgstr ""
 
 #: geonode/templates/index.html:23
-msgid "http://docs.geonode.org/en/master/tutorials/users/index.html"
+msgid "http://docs.geonode.org/en/master/usage"
 msgstr ""
 
 #: geonode/templates/index.html:23

diff --git a/geonode/tests/integration.py b/geonode/tests/integration.py
@@ -104,10 +104,9 @@ def zip_dir(basedir, archivename):
 r"""
  HOW TO RUN THE TESTS
  --------------------
- (https://github.com/GeoNode/geonode/blob/master/docs/tutorials/devel/testing.txt)
 
  1)
-  (https://github.com/GeoNode/geonode/blob/master/docs/tutorials/devel/envsetup/paver.txt)
+  (http://docs.geonode.org/en/master/install/core/index.html?highlight=paver#run-geonode-for-the-first-time-in-debug-mode)
 
   $ paver setup
 

diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -0,0 +1 @@
+ipdb
diff --git a/scripts/docker/env/development/celery.env b/scripts/docker/env/development/celery.env
@@ -0,0 +1,43 @@
+DJANGO_SETTINGS_MODULE=geonode.settings
+GEONODE_INSTANCE_NAME=geonode
+GEONODE_LB_HOST_IP
+GEONODE_LB_PORT
+DEFAULT_BACKEND_DATASTORE=datastore
+GEONODE_DATABASE=geonode
+GEONODE_DATABASE_PASSWORD=geonode
+GEONODE_GEODATABASE=geonode_data
+GEONODE_GEODATABASE_PASSWORD=geonode_data
+ASYNC_SIGNALS=True
+BROKER_URL=amqp://guest:guest@rabbitmq:5672
+DOCKER_ENV=development
+IS_CELERY=True
+DEBUG=True
+C_FORCE_ROOT=1
+SITEURL=http://localhost/
+# replaced with defaults in settings
+GEOSERVER_PUBLIC_LOCATION=http://localhost/geoserver/
+GEOSERVER_WEB_UI_LOCATION=http://localhost/geoserver/
+GEOSERVER_LOCATION=http://geoserver:8080/geoserver/
+OGC_REQUEST_TIMEOUT=300
+STATIC_ROOT=/mnt/volumes/statics/static/
+MEDIA_ROOT=/mnt/volumes/statics/uploaded/
+GEOIP_PATH=/mnt/volumes/statics/geoip.db
+ALLOWED_HOSTS=['django', '*']
+ADMIN_EMAILS
+DEFAULT_BACKEND_UPLOADER=geonode.importer
+TIME_ENABLED=True
+MOSAIC_ENABLED=False
+GEOGIG_ENABLED=False
+HAYSTACK_SEARCH=False
+HAYSTACK_ENGINE_URL=http://elasticsearch:9200/
+HAYSTACK_ENGINE_INDEX_NAME=haystack
+HAYSTACK_SEARCH_RESULTS_PER_PAGE=200
+
+
+ALLOWED_DOCUMENT_TYPES=['doc', 'docx', 'gif', 'jpg', 'jpeg', 'ods', 'odt', 'odp', 'pdf', 'png','ppt', 'pptx', 'rar', 'sld', 'tif', 'tiff', 'txt', 'xls', 'xlsx', 'xml', 'zip', 'gz', 'qml']
+MAX_DOCUMENT_SIZE=2
+
+# GEOSERVER_ADMIN_PASSWORD=admin
+# See https://github.com/geosolutions-it/geonode-generic/issues/28
+# to see why we force API version to 1.24
+DOCKER_API_VERSION="1.24"
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		COMPOSE_PROJECT_NAME=geonode
		SET_DOCKER_ENV=production