bird-house · mishaschwartz · Jul 31, 2023 · Aug 1, 2023 · Aug 1, 2023 · Aug 2, 2023
@@ -3,6 +3,7 @@ birdhouse/env.local
 birdhouse/certkey.pem
 birdhouse/cert.pem
 birdhouse/key.pem
+birdhouse/build/
 
 ## Deployment
 vagrant_variables.yml

@@ -15,7 +15,22 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Write build context to a build directory instead of using `birdhouse/`:
+  - Links all files needed to run `docker compose` commands to a build directory
+  - Generates a unified `docker-compose.yml` file that can be easily inspected to see exactly what is being deployed.
+  - Writes the generated output of template files to a new directory instead of beside the original template file
+    - this will keep the repo cleaner and won't require keeping legacy files in the various `.gitignore` file peppered
+      throughout the repository.
+  - The build directory is in `birdhouse/build` by default but can be changed by setting the `BUILD_DIR` variable in
+    `env.local`.
+  - No longer requires that custom component config files be placed relative to `./birdhouse/docker-compose.yml` since
+    these files are copied to the relevant location in the build directory.
+    - previously, file paths in `docker-compose-extra.yml` files were either absolute or relative paths from the
+      `birdhouse/` directory.
+    - now, they are absolute or relative paths from the parent of the component's directory (this resolves to the build
+      directory when the stack is started up).
 
 [1.33.5](https://github.com/bird-house/birdhouse-deploy/tree/1.33.5) (2023-10-02)
 ------------------------------------------------------------------------------------------------------------------

@@ -0,0 +1,188 @@
+# This file contains files to ignore that were generated when writing the result of template files
+# to the same directory as the template file itself. This is no longer necessary since all dynamically
+# generated files are written to the BUILD_DIR directory.
+# This .gitignore is kept for backwards compatibility so that if a deployment still contains these files
+# they will still be ignored properly.
+# This .gitignore file may be removed at a later date.
+components/cowbird/conf.extra-service.d/cowbird.conf
+components/cowbird/config/canarie-api/canarie_api_monitoring.py
+components/cowbird/config/canarie-api/cowbird_config.py
+components/cowbird/config/cowbird/celeryconfig.py
+components/cowbird/config/cowbird/config.yml
+components/cowbird/config/cowbird/cowbird.ini
+components/cowbird/config/magpie/config.yml
+components/cowbird/config/proxy/canarie_api_monitoring.py
+components/cowbird/config/proxy/conf.extra-service.d/cowbird.conf
+components/monitoring/alertmanager.yml
+components/monitoring/config/magpie/config.yml
+components/monitoring/config/proxy/conf.extra-service.d/monitoring.conf
+components/monitoring/grafana_dashboards.yml
+components/monitoring/grafana_datasources.yml
+components/monitoring/prometheus.rules
+components/monitoring/prometheus.yml
+components/scheduler/config.yml
+components/weaver/conf.extra-service.d/weaver.conf
+components/weaver/config/canarie-api/canarie_api_monitoring.py
+components/weaver/config/canarie-api/weaver_config.py
+components/weaver/config/magpie/config.yml
+components/weaver/config/proxy/canarie_api_monitoring.py
+components/weaver/config/proxy/conf.extra-service.d/weaver.conf
+components/weaver/config/weaver/data_sources.yml
+components/weaver/config/weaver/request_options.yml
+components/weaver/config/weaver/weaver.ini
+components/weaver/config/weaver/wps_processes.yml
+components/weaver/service-config.json
+config/canarie-api/conf.extra-service.d/canarie-api.conf
+config/canarie-api/config/canarie-api/conf.extra-service.d/canarie-api.conf
+config/canarie-api/config/proxy/conf.extra-service.d/canarie-api.conf
+config/canarie-api/docker_configuration.py
+config/catalog/
+config/finch/config/canarie-api/canarie_api_monitoring.py
+config/finch/config/magpie/providers.cfg
+config/finch/config/proxy/canarie_api_monitoring.py
+config/finch/service-config.json
+config/finch/wps.cfg
+deprecated-components/flyingpigeon/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/flyingpigeon/config/magpie/permissions.cfg
+deprecated-components/flyingpigeon/config/magpie/providers.cfg
+deprecated-components/flyingpigeon/service-config.json
+deprecated-components/flyingpigeon/wps.cfg
+config/frontend/
+config/geoserver/config/canarie-api/canarie_api_monitoring.py
+config/geoserver/config/magpie/providers.cfg
+config/geoserver/config/proxy/canarie_api_monitoring.py
+config/geoserver/config/proxy/conf.extra-service.d/geoserver.conf
+config/geoserver/geoserver_canarie_api_monitoring.py
+config/geoserver/geoserver-magpie-provider.cfg
+config/geoserver/service-config.json
+config/hummingbird/config/canarie-api/canarie_api_monitoring.py
+config/hummingbird/config/magpie/providers.cfg
+config/hummingbird/custom.cfg
+config/hummingbird/service-config.json
+config/jupyterhub/config/canarie-api/canarie_api_monitoring.py
+config/jupyterhub/config/proxy/canarie_api_monitoring.py
+config/jupyterhub/config/proxy/conf.extra-service.d/jupyterhub.conf
+config/jupyterhub/custom_templates/login.html
+config/jupyterhub/jupyterhub_canarie_api_monitoring.py
+config/jupyterhub/jupyterhub_config.py
+config/jupyterhub/service-config.json
+config/magpie/config/canarie-api/canarie_api_monitoring.py
+config/magpie/config/proxy/canarie_api_monitoring.py
+config/magpie/config/proxy/conf.extra-service.d/magpie.conf
+config/magpie/docker-entrypoint-initdb.d/create-db.sql
+config/magpie/magpie_canarie_api_monitoring.py
+config/magpie/magpie.ini
+config/magpie/permissions.cfg
+config/magpie/postgres-credentials.env
+config/magpie/providers.cfg
+config/malleefowl/
+config/ncops/
+config/ncwms2/
+config/phoenix/
+config/portainer/config/proxy/conf.extra-service.d/portainer.conf
+config/postgres/credentials.env
+config/postgres-magpie/
+config/project-api/
+config/proxy/conf.d/all-services.include
+config/proxy/conf.d/frontend.conf
+config/proxy/conf.d/lb_catalog.conf
+config/proxy/conf.d/lb_emu.conf
+config/proxy/conf.d/lb_flyingpigeon.conf
+config/proxy/conf.d/lb_malleefowl.conf
+config/proxy/conf.d/lb_ncops.conf
+config/proxy/nginx.conf
+config/proxy/static/
+config/raven/config/canarie-api/canarie_api_monitoring.py
+config/raven/config/magpie/providers.cfg
+config/raven/config/proxy/canarie_api_monitoring.py
+config/raven/service-config.json
+config/raven/wps.cfg
+config/solr/
+config/thredds/catalog.xml
+config/thredds/config/canarie-api/canarie_api_monitoring.py
+config/thredds/config/magpie/providers.cfg
+config/thredds/config/proxy/canarie_api_monitoring.py
+config/thredds/config/proxy/conf.extra-service.d/thredds.conf
+config/thredds/service-config.json
+config/thredds/threddsConfig.xml
+config/twitcher/config/canarie-api/canarie_api_monitoring.py
+config/twitcher/config/proxy/canarie_api_monitoring.py
+config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf
+config/twitcher/twitcher.ini
+config/wps_outputs-volume/config/canarie-api/canarie_api_monitoring.py
+config/wps_outputs-volume/config/proxy/canarie_api_monitoring.py
+config/wps_outputs-volume/config/proxy/conf.extra-service.d/wps_outputs-volume.conf
+deprecated-components/catalog/catalog.cfg
+deprecated-components/catalog/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/catalog/config/magpie/permissions.cfg
+deprecated-components/catalog/config/magpie/providers.cfg
+deprecated-components/catalog/config/proxy/canarie_api_monitoring.py
+deprecated-components/frontend/conf.extra-service.d/frontend.conf
+deprecated-components/frontend/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/frontend/config/proxy/canarie_api_monitoring.py
+deprecated-components/frontend/config/proxy/conf.extra-service.d/frontend.conf
+deprecated-components/frontend/frontend.env
+deprecated-components/malleefowl/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/malleefowl/config/magpie/providers.cfg
+deprecated-components/malleefowl/config/proxy/canarie_api_monitoring.py
+deprecated-components/malleefowl/custom.cfg
+deprecated-components/malleefowl/malleefowl_canarie_api_monitoring.py
+deprecated-components/ncops/ncops.cfg
+deprecated-components/ncwms2/config/magpie/providers.cfg
+deprecated-components/ncwms2/config/proxy/conf.extra-service.d/ncwms2.conf
+deprecated-components/ncwms2/custom.cfg
+deprecated-components/ncwms2/ncwms2-magpie-provider.cfg
+deprecated-components/phoenix/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/phoenix/config/proxy/canarie_api_monitoring.py
+deprecated-components/phoenix/custom.cfg
+deprecated-components/project-api/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/project-api/config/proxy/canarie_api_monitoring.py
+deprecated-components/project-api/config/proxy/conf.extra-service.d/project-api.conf
+deprecated-components/solr/config/canarie-api/canarie_api_monitoring.py
+deprecated-components/solr/config/proxy/canarie_api_monitoring.py
+optional-components/canarie-api-full-monitoring/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/canarie-api/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/catalog/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/cowbird/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/finch/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/hummingbird/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/malleefowl/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/ncwms2/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/proxy/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/raven/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/thredds/canarie_api_full_monitoring.py
+optional-components/canarie-api-full-monitoring/config/weaver/canarie_api_full_monitoring.py
+optional-components/emu/config/canarie-api/canarie_api_monitoring.py
+optional-components/emu/config/magpie/permissions.cfg
+optional-components/emu/config/magpie/providers.cfg
+optional-components/emu/config/proxy/canarie_api_monitoring.py
+optional-components/emu/emu_canarie_api_monitoring.py
+optional-components/emu/emu-magpie-permission.cfg
+optional-components/emu/emu-magpie-provider.cfg
+optional-components/emu/wps.cfg
+optional-components/generic_bird/config/canarie-api/canarie_api_monitoring.py
+optional-components/generic_bird/config/magpie/permissions.cfg
+optional-components/generic_bird/config/magpie/providers.cfg
+optional-components/generic_bird/config/proxy/canarie_api_monitoring.py
+optional-components/generic_bird/generic_bird_canarie_api_monitoring.py
+optional-components/generic_bird/generic_bird-magpie-permission.cfg
+optional-components/generic_bird/generic_bird-magpie-provider.cfg
+optional-components/generic_bird/postgres/credentials.env
+optional-components/generic_bird/wps.cfg
+optional-components/secure-data-proxy/conf.extra-service.d/secure-data-auth.conf
+optional-components/secure-data-proxy/config/magpie/config.yml
+optional-components/secure-data-proxy/config/proxy/conf.extra-service.d/secure-data-auth.conf
+optional-components/test-geoserver-secured-access/config/magpie/permissions.cfg
+optional-components/test-geoserver-secured-access/config/magpie/providers.cfg
+optional-components/test-geoserver-secured-access/config/magpie/test-geoserver-permissions.cfg
+optional-components/test-geoserver-secured-access/config/magpie/test-geoserver-providers.cfg
+optional-components/test-geoserver-secured-access/config/proxy/conf.d/test-geoserver-secured.conf
+optional-components/testthredds/catalog.xml
+optional-components/testthredds/conf.extra-service.d/
+optional-components/testthredds/config/canarie-api/canarie_api_monitoring.py
+optional-components/testthredds/config/proxy/canarie_api_monitoring.py
+optional-components/testthredds/config/proxy/conf.extra-service.d/testthredds-service.conf
+optional-components/testthredds/testthredds_canarie_api_monitoring.py
+optional-components/testthredds/threddsConfig.xml
+optional-components/test-weaver/config/magpie/test-weaver-permissions.yml
+optional-components/test-weaver/config/weaver/request_options.yml
@@ -9,4 +9,4 @@ services:
       #   the Cowbird config, and then all python configuration within 'optional-components/canarie-api-full-monitoring'.
       #   It is important to load Cowbird before, as the full monitoring need to know if 'Cowbird' service is added to the
       #   stack in order to apply the corresponding public endpoint conditionally.
-      - ./components/cowbird/config/proxy/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/0_cowbird_config.py:ro
+      - ./cowbird/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/0_cowbird_config.py:ro
@@ -12,10 +12,10 @@ services:
       #   duplicate references for providers/permissions/webhooks config files. This is because 'MAGPIE_CONFIG_PATH' is
       #   not used to allow parsing additive per-component config files for each extendable service, using loading of
       #   all configuration files found in mounted directories.
-      - ./components/cowbird/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/cowbird.yml:ro
-      - ./components/cowbird/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/cowbird.yml:ro
-      - ./components/cowbird/config/magpie/config.yml:${MAGPIE_WEBHOOKS_CONFIG_PATH}/cowbird.yml:ro
+      - ./cowbird/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/cowbird.yml:ro
+      - ./cowbird/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/cowbird.yml:ro
+      - ./cowbird/config/magpie/config.yml:${MAGPIE_WEBHOOKS_CONFIG_PATH}/cowbird.yml:ro
   cowbird-worker:
     env_file:
       # user/password for admin session to perform permission sync updates
-      - ./config/magpie/postgres-credentials.env
+      - ./magpie/postgres-credentials.env
@@ -4,6 +4,6 @@ services:
   # extend proxy with endpoint and config for Cowbird API access
   proxy:
     volumes:
-      - ./components/cowbird/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/cowbird:ro
+      - ./cowbird/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/cowbird:ro
     links:
       - cowbird
@@ -38,10 +38,10 @@ services:
     depends_on:
       - cowbird-mongodb
     volumes:
-      - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
-      - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
+      - ./cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
+      - ./cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
       # even if not running tasks here, they must be registered to send them off to the right place!
-      - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
+      - ./cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
       - "${DATA_PERSIST_SHARED_ROOT}:${DATA_PERSIST_SHARED_ROOT}"
     restart: always
     logging: *default-logging
@@ -68,9 +68,9 @@ services:
       - cowbird-mongodb
       - cowbird  # if not started first, sometimes celery misbehaves and will not pick jobs in queue
     volumes:
-      - ./components/cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
-      - ./components/cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
-      - ./components/cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
+      - ./cowbird/config/cowbird/config.yml:/opt/local/src/cowbird/config/cowbird.yml
+      - ./cowbird/config/cowbird/cowbird.ini:/opt/local/src/cowbird/config/cowbird.ini
+      - ./cowbird/config/cowbird/celeryconfig.py:/opt/local/src/cowbird/config/celeryconfig.py
       - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}"
     restart: always
     logging: *default-logging

@@ -3,5 +3,5 @@ version: "3.4"
 services:
   magpie:
     volumes:
-      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/monitoring.yml:ro
-      - ./components/monitoring/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/monitoring.yml:ro
+      - ./monitoring/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/monitoring.yml:ro
+      - ./monitoring/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/monitoring.yml:ro
@@ -3,4 +3,4 @@ version: "3.4"
 services:
   proxy:
     volumes:
-      - ./components/monitoring/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/monitoring:ro
+      - ./monitoring/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/monitoring:ro
@@ -33,8 +33,8 @@ services:
     image: prom/prometheus:v2.19.0
     container_name: prometheus
     volumes:
-      - ./components/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ./components/monitoring/prometheus.rules:/etc/prometheus/prometheus.rules:ro
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus.rules:/etc/prometheus/prometheus.rules:ro
       - prometheus_persistence:/prometheus:rw
     command:
       # restore original CMD from image
@@ -55,9 +55,9 @@ services:
     image: grafana/grafana:7.0.3
     container_name: grafana
     volumes:
-      - ./components/monitoring/grafana_datasources.yml:/etc/grafana/provisioning/datasources/grafana_datasources.yml:ro
-      - ./components/monitoring/grafana_dashboards.yml:/etc/grafana/provisioning/dashboards/grafana_dashboards.yml:ro
-      - ./components/monitoring/grafana_dashboards:/etc/grafana/dashboards:ro
+      - ./monitoring/grafana_datasources.yml:/etc/grafana/provisioning/datasources/grafana_datasources.yml:ro
+      - ./monitoring/grafana_dashboards.yml:/etc/grafana/provisioning/dashboards/grafana_dashboards.yml:ro
+      - ./monitoring/grafana_dashboards:/etc/grafana/dashboards:ro
       - grafana_persistence:/var/lib/grafana:rw
     environment:
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
@@ -73,8 +73,8 @@ services:
     image: prom/alertmanager:v0.21.0
     container_name: alertmanager
     volumes:
-      - ./components/monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
-      - ./components/monitoring/alertmanager.tmpl:/etc/alertmanager/template/default.tmpl:ro
+      - ./monitoring/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - ./monitoring/alertmanager.tmpl:/etc/alertmanager/template/default.tmpl:ro
       - alertmanager_persistence:/alertmanager:rw
     command:
       # restore original CMD from image

@@ -6,7 +6,7 @@ services:
     container_name: scheduler
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./components/scheduler/config.yml:/opt/crontab/config.yml:ro
+      - ./scheduler/config.yml:/opt/crontab/config.yml:ro
     environment:
       COMPOSE_DIR: ${PWD}
       AUTODEPLOY_DEPLOY_KEY_ROOT_DIR: ${AUTODEPLOY_DEPLOY_KEY_ROOT_DIR}