Merge branch 'main' into 182-add-local-path

.gitignore

@@ -6,3 +6,4 @@ node_modules
 /.eslintcache
 /tests/data/json_docs
 /deploy/cht_sync/values.yaml
+/bastion/authorized_keys

CHANGELOG.md

@@ -1,3 +1,38 @@
+## [1.3.3](https://github.com/medic/cht-sync/compare/v1.3.2...v1.3.3) (2025-01-06)
+
+
+### Bug Fixes
+
+* **#198:** add configurable POSTGRES_PORT to docker compose file ([#199](https://github.com/medic/cht-sync/issues/199)) ([c286014](https://github.com/medic/cht-sync/commit/c286014ea45210604e601d4a8b02b447b108b853)), closes [#198](https://github.com/medic/cht-sync/issues/198)
+
+## [1.3.2](https://github.com/medic/cht-sync/compare/v1.3.1...v1.3.2) (2024-12-12)
+
+
+### Bug Fixes
+
+* **#189:** add dbt thread count to helm ([#195](https://github.com/medic/cht-sync/issues/195)) ([e545b04](https://github.com/medic/cht-sync/commit/e545b0477441422ef82bc3a68ebcbeb1275d1ca1)), closes [#189](https://github.com/medic/cht-sync/issues/189) [#189](https://github.com/medic/cht-sync/issues/189) [#189](https://github.com/medic/cht-sync/issues/189)
+
+## [1.3.1](https://github.com/medic/cht-sync/compare/v1.3.0...v1.3.1) (2024-12-11)
+
+
+### Bug Fixes
+
+* **#193:** only create indexes if table does not exist ([#194](https://github.com/medic/cht-sync/issues/194)) ([0e6b671](https://github.com/medic/cht-sync/commit/0e6b6710aa85be45039c16351d451843bdcec527)), closes [#193](https://github.com/medic/cht-sync/issues/193) [#193](https://github.com/medic/cht-sync/issues/193)
+
+# [1.3.0](https://github.com/medic/cht-sync/compare/v1.2.0...v1.3.0) (2024-12-06)
+
+
+### Features
+
+* make dbt thread count configurable ([#190](https://github.com/medic/cht-sync/issues/190)) ([17318c1](https://github.com/medic/cht-sync/commit/17318c1fe6cf7529127c866dba8402914dd82fac))
+
+# [1.2.0](https://github.com/medic/cht-sync/compare/v1.1.4...v1.2.0) (2024-12-05)
+
+
+### Features
+
+* **#174:** bastion Dockerfile and compose file ([#177](https://github.com/medic/cht-sync/issues/177)) ([27d1739](https://github.com/medic/cht-sync/commit/27d1739041cf55b02a6257d4f1f65779d83b2737)), closes [#174](https://github.com/medic/cht-sync/issues/174) [#174](https://github.com/medic/cht-sync/issues/174)
+
 ## [1.1.4](https://github.com/medic/cht-sync/compare/v1.1.3...v1.1.4) (2024-10-31)
 
 

bastion/Dockerfile

@@ -0,0 +1,31 @@
+# NOSONAR for "don't run docker images as root" - because sshd requires root https://superuser.com/a/1548482 :(
+# Deep link to Sonar "safe": https://sonarcloud.io/project/security_hotspots?status=SAFE&pullRequest=177&id=medic_cht-sync&tab=activity
+
+FROM alpine:3.20
+
+ARG HOME=/var/lib/bastion
+
+ARG USER=bastion
+ARG GROUP=bastion
+ARG UID=4096
+ARG GID=4096
+
+ENV HOST_KEYS_PATH_PREFIX="/usr"
+ENV HOST_KEYS_PATH="${HOST_KEYS_PATH_PREFIX}/etc/ssh"
+
+COPY bastion /usr/sbin/bastion
+
+RUN addgroup -S -g ${GID} ${GROUP} \
+    && adduser -D -h ${HOME} -s /bin/false -g "${USER} service" \
+           -u ${UID} -G ${GROUP} ${USER} \
+    && sed -i "s/${USER}:!/${USER}:*/g" /etc/shadow \
+    && set -x \
+    && apk add --no-cache openssh-server curl \
+    && echo -e "\nLogin Successful! \n\nHowever, interactive sessions not allowed. Use \"-N\" with ssh tunnel command instead: \n" > /etc/motd \
+    && echo -e "\tssh -N -L 5432:CONTAINER-NAME:5432 bastion@PUBLIC-IP-OR-DNS -p 22222\n" >> /etc/motd \
+    && chmod +x /usr/sbin/bastion \
+    && mkdir -p ${HOST_KEYS_PATH} \
+    && mkdir /etc/ssh/auth_principals \
+    && echo "bastion" > /etc/ssh/auth_principals/bastion
+
+ENTRYPOINT ["bastion"]

bastion/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Mark
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bastion/README.md

@@ -0,0 +1,9 @@
+This project hard forked from [docker-bastion](https://github.com/binlab/docker-bastion/tree/master) at version [1.2.0](https://github.com/binlab/docker-bastion/releases/tag/v1.2.0)
+
+Per MIT license, copyright of this `bastion` sub-directory is Mark/binlab/mark.binlab@gmail.com and MIT license file persists.
+
+Sample SSH tunnel to connect to container called `cht-sync-postgres-1` on remote server with IP `44.33.22.11`:
+
+```shell
+ssh -N -L 5432:cht-sync-postgres-1:5432 bastion@44.33.22.11 -p 22222
+```

bastion/authorized_keys.example

@@ -0,0 +1 @@
+# add external users' keys here

bastion/bastion

@@ -0,0 +1,52 @@
+#!/usr/bin/env sh
+
+HOST_KEYS_PATH_PREFIX="${HOST_KEYS_PATH_PREFIX:='/'}"
+HOST_KEYS_PATH="${HOST_KEYS_PATH:='/etc/ssh'}"
+
+if [ -n "$TRUSTED_USER_CA_KEYS" ]; then
+    CONFIG_TRUSTED_USER_CA_KEYS="-o TrustedUserCAKeys=$TRUSTED_USER_CA_KEYS"
+    CONFIG_AUTHORIZED_PRINCIPALS_FILE="-o AuthorizedPrincipalsFile=/etc/ssh/auth_principals/%u"
+fi
+
+if [ ! -f "$HOST_KEYS_PATH/ssh_host_rsa_key" ]; then
+    /usr/bin/ssh-keygen -A -f "$HOST_KEYS_PATH_PREFIX"
+fi
+
+if [ -n "$LISTEN_ADDRESS" ]; then
+    CONFIG_LISTEN_ADDRESS="-o ListenAddress=$LISTEN_ADDRESS"
+else
+    CONFIG_LISTEN_ADDRESS="-o ListenAddress=0.0.0.0"
+fi
+
+if [ -n "$LISTEN_PORT" ]; then
+    CONFIG_LISTEN_PORT="-o Port=$LISTEN_PORT"
+else
+    CONFIG_LISTEN_PORT="-o Port=22"
+fi
+
+# todo - original project has this as "assumes your authorized_keys file with 644 permissions
+# and mounted under /var/lib/bastion/authorized_keys." - but this simply doesn't work
+# without this hack
+cp /var/lib/bastion/authorized_keys-tmp /var/lib/bastion/authorized_keys
+chown bastion /var/lib/bastion/authorized_keys
+chmod 600 /var/lib/bastion/authorized_keys
+
+/usr/sbin/sshd -D -e -4 \
+    -o "HostKey=$HOST_KEYS_PATH/ssh_host_rsa_key" \
+    -o "HostKey=$HOST_KEYS_PATH/ssh_host_ecdsa_key" \
+    -o "HostKey=$HOST_KEYS_PATH/ssh_host_ed25519_key" \
+    -o "PasswordAuthentication=no" \
+    -o "PermitEmptyPasswords=no" \
+    -o "PermitRootLogin=no" \
+    -o "X11Forwarding=no" \
+    -o "AllowAgentForwarding=yes" \
+    -o "GatewayPorts=yes" \
+    -o "PermitTunnel=yes" \
+    -o "PubkeyAuthentication=yes" \
+    -o "AllowTcpForwarding=yes" \
+    -o "AuthorizedKeysFile=/var/lib/bastion/authorized_keys" \
+    -o "AuthorizedKeysCommandUser=nobody" \
+    $CONFIG_TRUSTED_USER_CA_KEYS \
+    $CONFIG_AUTHORIZED_PRINCIPALS_FILE \
+    $CONFIG_LISTEN_ADDRESS \
+    $CONFIG_LISTEN_PORT

couch2pg/src/setup.js

@@ -18,11 +18,11 @@ CREATE TABLE IF NOT EXISTS ${db.postgresProgressTable} (
 );`;
 
 const createDeleteIndex = `
-CREATE INDEX IF NOT EXISTS _deleted ON ${db.postgresTable}(_deleted);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS _deleted ON ${db.postgresTable}(_deleted);
 `;
 
 const createTimestampIndex = `
-CREATE INDEX IF NOT EXISTS saved_timestamp ON ${db.postgresTable}(saved_timestamp);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS saved_timestamp ON ${db.postgresTable}(saved_timestamp);
 `;
 
 export const createDatabase = async () => {

couch2pg/tests/unit/setup.spec.js

@@ -43,10 +43,10 @@ CREATE TABLE IF NOT EXISTS v1.whatever (
   doc jsonb
 )`]);
       expect(pgClient.query.args[2]).to.deep.equal([`
-CREATE INDEX IF NOT EXISTS _deleted ON v1.whatever(_deleted);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS _deleted ON v1.whatever(_deleted);
 `]);
       expect(pgClient.query.args[3]).to.deep.equal([`
-CREATE INDEX IF NOT EXISTS saved_timestamp ON v1.whatever(saved_timestamp);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS saved_timestamp ON v1.whatever(saved_timestamp);
 `]);
       expect(pgClient.query.args[4]).to.deep.equal([`
 CREATE TABLE IF NOT EXISTS v1.couchdb_progress (

dbt/.dbt/profiles.yml

@@ -2,6 +2,7 @@ default:
   target: default
   outputs:
     default:
+      threads: "{{ env_var('DBT_THREAD_COUNT') | as_number }}"
       type: postgres
       host: "{{ env_var('POSTGRES_HOST') }}"
       user: "{{ env_var('POSTGRES_USER') }}"

deploy/cht_sync/templates/dbt.yaml

@@ -39,4 +39,6 @@ spec:
               value: {{ .Values.postgres.schema }}
             - name: CHT_PIPELINE_BRANCH_URL
               value: {{ .Values.cht_pipeline_branch_url }}
+            - name: DBT_THREAD_COUNT
+              value: {{ .Values.dbt_thread_count | default "1" | quote }}
 

deploy/cht_sync/values.yaml.template

@@ -10,6 +10,7 @@ postgres:
   port: 5432 # if postgres is outside the cluster
 
 cht_pipeline_branch_url: "https://github.com/medic/cht-pipeline.git#main"
+dbt_thread_count: 1
 
 # values shared by all couchdb instances
 # can be omitted if couchdb instances do not share any values

docker-compose.bastion.yml

@@ -0,0 +1,12 @@
+services:
+  bastion:
+    build: ./bastion/
+    restart: unless-stopped
+    ports:
+      - ${BASTION_PORT:-22222}:22/tcp
+    volumes:
+      - ${BASTION_AUTHORIZED_KEYS_FILE:-$PWD/bastion/authorized_keys}:/var/lib/bastion/authorized_keys-tmp
+      - bastion:/usr/etc/ssh:rw
+
+volumes:
+  bastion:

docker-compose.couchdb.yml

@@ -1,11 +1,12 @@
-version: '3.7'
-
 services:
   couchdb:
-    image: public.ecr.aws/medic/cht-couchdb:4.8.0
+    image: public.ecr.aws/medic/cht-couchdb:4.15.0
     restart: always
     ports:
       - "5984:5984"
     environment:
       - COUCHDB_USER=${COUCHDB_USER}
       - COUCHDB_PASSWORD=${COUCHDB_PASSWORD}
+      - COUCHDB_SECRET=9c0d6034-0f19-45df-8fcd-5fec9c473c73
+      - COUCHDB_UUID=4c6bffc8-a5ac-4f98-a34c-6fb0e63964e5
+      - SVC_NAME=couchdb

docker-compose.pgadmin.yml

@@ -0,0 +1,13 @@
+services:
+  postgres:
+    ports:
+      - 5432:${POSTGRES_PORT:-5432}
+
+  pgadmin:
+    image: dpage/pgadmin4
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
+      PGADMIN_CONFIG_SERVER_MODE: 'False'
+    ports:
+      - "${PGADMIN_PORT:-5050}:80"

docker-compose.postgres.yml

@@ -2,22 +2,11 @@ services:
   postgres:
     image: postgres:16
     restart: always
-    ports:
-      - 5432:5432
     volumes:
       - ./postgres/init-dbt-resources.sh:/docker-entrypoint-initdb.d/init-dbt-resources.sh:z
     environment:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_TABLES=${COUCHDB_DBS}
-      - POSTGRES_SCHEMA=${POSTGRES_SCHEMA}
-
-  pgadmin:
-    image: dpage/pgadmin4
-    environment:
-      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin4@pgadmin.org}
-      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin}
-      PGADMIN_CONFIG_SERVER_MODE: 'False'
-    ports:
-      - "${PGADMIN_PORT:-5050}:80"
+      - POSTGRES_SCHEMA=${POSTGRES_SCHEMA}

docker-compose.yml

@@ -19,7 +19,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_HOST=${POSTGRES_HOST}
       - POSTGRES_DB=${POSTGRES_DB}
-      - POSTGRES_PORT=5432
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
       - POSTGRES_SCHEMA=${POSTGRES_SCHEMA}
       - POSTGRES_TABLE=${POSTGRES_TABLE}
     restart: always
@@ -29,6 +29,7 @@ services:
     working_dir: /dbt/
     environment:
       - POSTGRES_HOST=${POSTGRES_HOST}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${POSTGRES_DB}
@@ -38,3 +39,4 @@ services:
       - CHT_PIPELINE_BRANCH_URL=${CHT_PIPELINE_BRANCH_URL}
       - DATAEMON_INTERVAL=${DATAEMON_INTERVAL}
       - DBT_PACKAGE_TARBALL_URL=${DBT_PACKAGE_TARBALL_URL}
+      - DBT_THREAD_COUNT=${DBT_THREAD_COUNT}

env.template

@@ -10,6 +10,7 @@ POSTGRES_PORT=5432
 # dbt
 CHT_PIPELINE_BRANCH_URL="https://github.com/medic/cht-pipeline.git#main"
 DATAEMON_INTERVAL=5
+DBT_THREAD_COUNT=1
 
 # if running locally, path to pipeline
 DBT_LOCAL_PATH="/path/to/cht-pipeline/"
@@ -24,3 +25,6 @@ COUCHDB_SECURE=false
 
 # (Optional) project wide
 #COMPOSE_PROJECT_NAME=cht-sync
+
+#BASTION_PORT=22222 # default is 22222 uncomment to change
+#BASTION_AUTHORIZED_KEYS_FILE=  # uncomment to change