diff --git a/.gitignore b/.gitignore index 4c49bd7..97aca2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .env +node_modules \ No newline at end of file diff --git a/Makefile b/Makefile index 26cb510..c2de018 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,10 @@ local: down extract-test-data build up prod: down build - docker-compose up logstash postgrest dbt + docker-compose -f docker-compose.couchdb.yml -f docker-compose.postgrest.yml -f docker-compose.yml \ + up logstash postgrest dbt gamma: down build COUCHDB_HOST=adp-sandbox.dev.medicmobile.org COUCHDB_DB=medic COUCHDB_USER=medic \ - docker-compose up logstash postgres postgrest dbt + docker-compose -f docker-compose.couchdb.yml -f docker-compose.postgres.yml -f docker-compose.yml \ + up logstash postgres postgrest dbt diff --git a/README.md b/README.md index a19872d..b1de6d1 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ make local The gamma environment setup involves starting Logstash, PostgreSQL, PostgREST, and DBT. This configuration facilitates data synchronization, transformation, and storage for medic gamma hosting. The required environment variables can be found in the `env.template` file, which should be customized accordingly for the specific deployment needs. +1. Update the following environment variables in your `.env` file: + +``` +# project wide: optional +COMPOSE_PROJECT_NAME=pipeline + +# couchdb and logstash: required environment variables for 'gamma', 'prod' and 'local' +COUCHDB_PASSWORD= +COUCHDB_SECURE=false +``` + +1. Start docker containers ```sh # starts: logstash, superset, postgres, postgrest, and dbt make gamma @@ -37,6 +49,38 @@ make gamma The production environment setup involves starting Logstash, PostgREST, and DBT. This configuration facilitates data synchronization, transformation, and storage for CHT production hosting. The required environment variables can be found in the `env.template` file, which should be customized accordingly for the specific deployment needs. +1. Update the following environment variables in your `.env` file: + +``` +# project wide: optional +COMPOSE_PROJECT_NAME=pipeline + +# postgrest and pogresql: required environment variables for 'gamma', prod and 'local' +POSTGRES_USER= +POSTGRES_PASSWORD= +POSTGRES_DB= +POSTGRES_TABLE= +POSTGRES_SCHEMA= + +# dbt: required environment variables for 'gamma', 'prod' and 'local' +DBT_POSTGRES_USER= +DBT_POSTGRES_PASSWORD= +DBT_POSTGRES_SCHEMA= +DBT_POSTGRES_HOST= # IP address + +# couchdb and logstash: required environment variables for 'gamma', 'prod' and 'local' +COUCHDB_PASSWORD= +COUCHDB_HOST= +COUCHDB_PORT= +COUCHDB_SECURE=false +``` + +1. (Optional) Start local version of PostgreSQL +``` +docker-compose -f docker-compose.postgres.yml -f docker-compose.yml up postgres +``` + +1. Start docker containers ```sh # starts: logstash, superset, postgrest and dbt make prod diff --git a/docker-compose.postgrest.yml b/docker-compose.postgrest.yml new file mode 100644 index 0000000..2696140 --- /dev/null +++ b/docker-compose.postgrest.yml @@ -0,0 +1,12 @@ +version: '3.7' + +services: + + postgrest: + image: postgrest/postgrest + restart: always + environment: + - PGRST_DB_URI=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:5432/${POSTGRES_DB} + - PGRST_DB_SCHEMA=${POSTGRES_SCHEMA} + # CHANGE FOR PROD!!!! + - PGRST_DB_ANON_ROLE=${POSTGRES_USER} diff --git a/docker-compose.yml b/docker-compose.yml index e58a942..7f81914 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: platform: linux/amd64 image: medicmobile/dataemon:latest environment: - - POSTGRES_HOST=postgres + - POSTGRES_HOST=${DBT_POSTGRES_HOST} - POSTGRES_USER=${DBT_POSTGRES_USER} - POSTGRES_PASSWORD=${DBT_POSTGRES_PASSWORD} - POSTGRES_DB=${POSTGRES_DB} diff --git a/env.template b/env.template index 6179923..cff6add 100644 --- a/env.template +++ b/env.template @@ -12,6 +12,7 @@ POSTGRES_SCHEMA=v1 DBT_POSTGRES_USER=dbt_user DBT_POSTGRES_PASSWORD=supercoolpassword DBT_POSTGRES_SCHEMA=dbt +DBT_POSTGRES_HOST=postgres # Your postgres instance IP or endpoint in "prod". CHT_PIPELINE_BRANCH_URL=https://github.com/medic/cht-pipeline.git#main # couchdb and logstash: required environment variables for 'gamma', 'prod' and 'local'