From 5fa27598c942ccda4e520b826c7f4d8397e55119 Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Wed, 5 Jan 2022 12:40:31 -0600 Subject: [PATCH 01/10] Optionally skip plv8 extension install The newer postgis/postgres docker images do not appear to have plv8 as an option. I cannot find a reference to use using plv8 (our sql functions use plpgsql) so I added a precondition to check if we are able to install, otherwise we continue. NOTE: all tests of the nldi services against this new image still pass. --- liquibase/changeLogs/postgres/nldi/changeLog.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/liquibase/changeLogs/postgres/nldi/changeLog.yml b/liquibase/changeLogs/postgres/nldi/changeLog.yml index 3526cfa..77a8c44 100644 --- a/liquibase/changeLogs/postgres/nldi/changeLog.yml +++ b/liquibase/changeLogs/postgres/nldi/changeLog.yml @@ -34,8 +34,13 @@ databaseChangeLog: - rollback: drop extension if exists postgis_tiger_geocoder; - changeSet: - author: drsteini + author: egrahn id: "create.extension.plv8" + preConditions: # check if the plv8 extension is available to install + - onFail: CONTINUE + - sqlCheck: + expectedResult: 1 + sql: select count(*) from pg_available_extensions where name = 'plv8' changes: - sql: create extension if not exists plv8; - rollback: drop extension if exists plv8; From 5a0550b638d9c4699b3e8d5bb9d62bd4ed7a0c56 Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Wed, 5 Jan 2022 12:41:14 -0600 Subject: [PATCH 02/10] Replace unsupported time function --- .../nldi/nhdplus_navigation/functions/navigate_cached.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liquibase/changeLogs/nldi/nhdplus_navigation/functions/navigate_cached.sql b/liquibase/changeLogs/nldi/nhdplus_navigation/functions/navigate_cached.sql index 220bcbf..820b5bd 100755 --- a/liquibase/changeLogs/nldi/nhdplus_navigation/functions/navigate_cached.sql +++ b/liquibase/changeLogs/nldi/nhdplus_navigation/functions/navigate_cached.sql @@ -95,7 +95,7 @@ BEGIN ,pstartcomid ,pMaxDistanceKm ,pstopcomid - ,(abstime(('now'::text)::timestamp(6) with time zone)) + ,now() ); END IF; From ebe1f000ea37412004b127501fb8620abadacecf Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Wed, 5 Jan 2022 12:45:31 -0600 Subject: [PATCH 03/10] Use maintained images to build CI image --- database/ci/Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/database/ci/Dockerfile b/database/ci/Dockerfile index c275319..a0baf27 100755 --- a/database/ci/Dockerfile +++ b/database/ci/Dockerfile @@ -1,7 +1,15 @@ -FROM usgswma/iow_test_db:postgis11-jre11 +FROM liquibase/liquibase:3.10 as Liquibase + +FROM postgis/postgis:12-3.0 AS Postgres LABEL maintainer=gs-w_eto_eb_federal_employees@usgs.gov +ENV LIQUIBASE_HOME /liquibase +ENV LIQUIBASE_WORKSPACE /liquibase +COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE} +ENV JAVA_HOME /usr/local/openjdk-11 +COPY --from=Liquibase ${JAVA_HOME} ${JAVA_HOME} + ############################################ # Grab files for initializing the database ############################################ @@ -11,5 +19,6 @@ COPY ./liquibase/scripts/*.sh /docker-entrypoint-initdb.d/ COPY ./liquibase/scripts/dbInit /docker-entrypoint-initdb.d/ COPY ./liquibase/scripts/dbCi /docker-entrypoint-initdb.d/ +RUN apt-get update && apt-get install -y curl RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/nhdplus.yahara.pgdump.gz" -o $LIQUIBASE_HOME/nhdplus.yahara.pgdump.gz RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/characteristic_data.yahara.pgdump.gz" -o $LIQUIBASE_HOME/characteristic_data.yahara.pgdump.gz From 1727fbaa664bee53e705a6b04f18bbce092f0d9d Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Wed, 5 Jan 2022 12:46:26 -0600 Subject: [PATCH 04/10] Update docker-compose syntax --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 17bdd0c..ac57b23 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3.3' services: - ciDB: - image: nldi_db_ci + ci: + image: nldi-db-ci build: context: . dockerfile: ./database/ci/Dockerfile @@ -23,10 +23,10 @@ services: - NLDI_DATABASE_ADDRESS=127.0.0.1 ports: - ${DB_CI_PORT}:5432 - container_name: ${NLDI_DATABASE_ADDRESS}_CI + container_name: nldi-db-ci - demoDB: - image: nldi_db_demo + demo: + image: nldi-db-demo build: context: . dockerfile: ./database/demo/Dockerfile @@ -47,10 +47,10 @@ services: - NLDI_DATABASE_ADDRESS=127.0.0.1 ports: - ${DB_DEMO_PORT}:5432 - container_name: ${NLDI_DATABASE_ADDRESS}_DEMO + container_name: nldi-db-demo db: - image: nldi_db + image: nldi-db build: context: . networks: @@ -61,10 +61,10 @@ services: - POSTGRES_DB=${NLDI_DATABASE_NAME} ports: - ${DB_PORT}:5432 - container_name: ${NLDI_DATABASE_ADDRESS} + container_name: nldi-db liquibase: - image: nldi_liquibase + image: nldi-liquibase depends_on: - db build: @@ -90,7 +90,7 @@ services: volumes: - ./liquibase/changeLogs:/home/java/workspace/ - ./liquibase/scripts/dbInit:/docker-entrypoint-initdb.d - container_name: nldi_liquibase + container_name: nldi-liquibase networks: nldi: From e175bad0fbae48898b83da48feb7326ef39f2e7b Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Wed, 5 Jan 2022 12:52:32 -0600 Subject: [PATCH 05/10] Add build args for docker image versions --- database/ci/Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/database/ci/Dockerfile b/database/ci/Dockerfile index a0baf27..af01982 100755 --- a/database/ci/Dockerfile +++ b/database/ci/Dockerfile @@ -1,6 +1,9 @@ -FROM liquibase/liquibase:3.10 as Liquibase +ARG LIQUIBASE_DOCKER_VERSION=3.10 +ARG POSTGIS_DOCKER_VERSION=12-3.0 -FROM postgis/postgis:12-3.0 AS Postgres +FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION} as Liquibase + +FROM postgis/postgis:${POSTGIS_DOCKER_VERSION} AS Postgres LABEL maintainer=gs-w_eto_eb_federal_employees@usgs.gov @@ -9,6 +12,7 @@ ENV LIQUIBASE_WORKSPACE /liquibase COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE} ENV JAVA_HOME /usr/local/openjdk-11 COPY --from=Liquibase ${JAVA_HOME} ${JAVA_HOME} +RUN apt-get update && apt-get install -y curl ############################################ # Grab files for initializing the database @@ -19,6 +23,5 @@ COPY ./liquibase/scripts/*.sh /docker-entrypoint-initdb.d/ COPY ./liquibase/scripts/dbInit /docker-entrypoint-initdb.d/ COPY ./liquibase/scripts/dbCi /docker-entrypoint-initdb.d/ -RUN apt-get update && apt-get install -y curl RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/nhdplus.yahara.pgdump.gz" -o $LIQUIBASE_HOME/nhdplus.yahara.pgdump.gz RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/characteristic_data.yahara.pgdump.gz" -o $LIQUIBASE_HOME/characteristic_data.yahara.pgdump.gz From 91e1058b9067a4edbc64db0e65f62451d2a70a54 Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Thu, 6 Jan 2022 10:17:16 -0600 Subject: [PATCH 06/10] Use maintained images in demo dockerfile --- database/demo/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/database/demo/Dockerfile b/database/demo/Dockerfile index 73504e8..8ea25c9 100755 --- a/database/demo/Dockerfile +++ b/database/demo/Dockerfile @@ -1,7 +1,19 @@ -FROM usgswma/iow_test_db:postgis11-jre11 +ARG LIQUIBASE_DOCKER_VERSION=3.10 +ARG POSTGIS_DOCKER_VERSION=12-3.0 + +FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION} as Liquibase + +FROM postgis/postgis:${POSTGIS_DOCKER_VERSION} AS Postgres LABEL maintainer=gs-w_eto_eb_federal_employees@usgs.gov +ENV LIQUIBASE_HOME /liquibase +ENV LIQUIBASE_WORKSPACE /liquibase +COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE} +ENV JAVA_HOME /usr/local/openjdk-11 +COPY --from=Liquibase ${JAVA_HOME} ${JAVA_HOME} +RUN apt-get update && apt-get install -y curl + ############################################ # Grab files for initializing the database ############################################ From 887a3d77f9722859b705eed4b7bdf3091962a32c Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Thu, 6 Jan 2022 10:51:51 -0600 Subject: [PATCH 07/10] Remove barebones docker image We can use postgis/postgis as the base since the usgswma docker hub images are no longer maintained. --- Dockerfile | 3 --- docker-compose.yml | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 938e37f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM usgswma/postgis-plv8:11-2.3.8 - -LABEL maintainer=gs-w_eto_eb_federal_employees@usgs.gov diff --git a/docker-compose.yml b/docker-compose.yml index ac57b23..51782af 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,9 +50,7 @@ services: container_name: nldi-db-demo db: - image: nldi-db - build: - context: . + image: postgis/postgis:12-3.0 networks: nldi: ipv4_address: ${DB_IPV4} From e729a1fc7e8eac2a6a9746a4d4029b0b40f0c3dc Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Thu, 6 Jan 2022 10:52:04 -0600 Subject: [PATCH 08/10] Remove travis-ci --- .travis.yml | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100755 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100755 index f6a2f97..0000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -language: bash - -services: - - docker - -env: - - POSTGRES_PASSWORD=changeMe - NLDI_DATABASE_ADDRESS=nldi_db_addr - NLDI_DATABASE_NAME=nldi - NLDI_DB_OWNER_USERNAME=nldi_db_owner - NLDI_DB_OWNER_PASSWORD=changeMe - NLDI_SCHEMA_OWNER_USERNAME=nldi_schema_owner - NLDI_SCHEMA_OWNER_PASSWORD=changeMe - NHDPLUS_SCHEMA_OWNER_USERNAME=nhdplus_schema_owner - NLDI_READ_ONLY_USERNAME=read_only_user - NLDI_READ_ONLY_PASSWORD=changeMe - LOCAL_NETWORK_NAME=nldi - DB_IPV4=172.26.0.2 - DB_PORT=5444 - LIQUIBASE_IPV4=172.26.0.3 - LIQUIBASE_VERSION=3.6.3 - JDBC_JAR=postgresql-42.2.5.jar - DB_CI_IPV4=172.26.0.4 - DB_CI_PORT=5445 - DB_DEMO_IPV4=172.26.0.5 - DB_DEMO_PORT=5446 - -install: true - -script: - - docker network create --subnet=172.26.0.0/16 nldi - - docker-compose up -d - - docker-compose ps - # Travis appears to only have a 16 char container name column. - - docker-compose ps | grep "nldi_db_addr\s.*Up" - - docker-compose ps | grep "nldi_db_addr_DEM.*Up" - - docker-compose ps | grep "nldi_db_addr_CI\s.*Up" - - docker-compose ps | grep "nldi_liquibase\s.*Exit 0\|nldi_liquibase\s.*Up" - - docker-compose stop - - docker network rm nldi From 2b0e36b26c460cbdcfc66262765cb9a51453a524 Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Thu, 6 Jan 2022 10:52:12 -0600 Subject: [PATCH 09/10] Update documentation --- README.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f3794f1..4f230eb 100755 --- a/README.md +++ b/README.md @@ -1,13 +1,11 @@ # NLDI Database Setup -[![Build Status](https://travis-ci.org/ACWI-SSWD/nldi-db.svg?branch=master)](https://travis-ci.org/ACWI-SSWD/nldi-db) - This repository contains Liquibase scripts for creating the NLDI PostGIS database. ## Docker Also included are Docker Compose scripts to: * Create PostgreSQL and Liquibase containers for testing the scripts. -* Create a continuous integration PostgreSQL database container. +* Create a continuous integration (CI) PostgreSQL database container. * Create a PostgreSQL database container for local development containing a sampling of data. ### Docker Network @@ -18,9 +16,7 @@ docker network create --subnet=172.26.0.0/16 nldi ``` ### Environment variables -In order to use the docker compose scripts, you will need to create a .env file in the project directory containing - -the following (shown are example values): +In order to use the docker compose scripts, you will need to create a .env file in the project directory containing the following (shown are example values): ``` POSTGRES_PASSWORD= @@ -98,30 +94,30 @@ The PostGIS database will be available on your localhost's port $DB_PORT, allowi ### CI Database ``` -docker-compose up ciDB +docker-compose up ci ``` It will be available on you localhost's port $DB_CI_PORT -You can also pull the image from Docker Hub and run it with +You can also pull the image from the GitHub Package Repository and run it with ``` -docker run -it --env-file ./.env -p 127.0.0.1:5445:5432 usgswma/wqp_db:ci +docker run -it --env-file ./.env -p 5445:5432 ghcr.io/acwi-sswd/nldi-db:ci-latest ``` where __./.env__ is the environment variable file you have locally and __5445__ can be changed to the port you wish to access it via. ### Demo Database ``` -docker-compose up demoDB +docker-compose up demo ``` It will be available on your localhost's port $DB_DEMO_PORT -You can also pull the image from Docker Hub and run it with +You can also pull the image from the GitHub Package Repository and run it with ``` -docker run -it --env-file ./.env -p 127.0.0.1:5446:5432/tcp usgswma/nldi-db:demo +docker run -it --env-file ./.env -p 5446:5432/tcp ghcr.io/acwi-sswd/nldi-db:demo-latest ``` where __./.env__ is the environment variable file you have locally and __5446__ can be changed to the port you wish to access it via. From a82973d51f94080b05229a7375ae1b3c6ce3aa0f Mon Sep 17 00:00:00 2001 From: Ethan Grahn Date: Thu, 13 Jan 2022 09:21:09 -0600 Subject: [PATCH 10/10] Make corrections to liquibase container --- docker-compose.yml | 13 ++++++------- liquibase/Dockerfile | 27 +++++++-------------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 51782af..1a60c02 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,8 @@ services: image: postgis/postgis:12-3.0 networks: nldi: - ipv4_address: ${DB_IPV4} + aliases: + - nldi-db environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${NLDI_DATABASE_NAME} @@ -67,12 +68,10 @@ services: - db build: context: ./liquibase - args: - - LIQUIBASE_VERSION=${LIQUIBASE_VERSION} - - A_JDBC_JAR=${JDBC_JAR} networks: nldi: - ipv4_address: ${LIQUIBASE_IPV4} + aliases: + - nldi-liquibase environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_DB=${NLDI_DATABASE_NAME} @@ -84,9 +83,9 @@ services: - NHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME} - NLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME} - NLDI_READ_ONLY_PASSWORD=${NLDI_READ_ONLY_PASSWORD} - - NLDI_DATABASE_ADDRESS=${NLDI_DATABASE_ADDRESS} + - NLDI_DATABASE_ADDRESS=nldi-db # using the network alias from the db service volumes: - - ./liquibase/changeLogs:/home/java/workspace/ + - ./liquibase/changeLogs:/liquibase/workspace/ - ./liquibase/scripts/dbInit:/docker-entrypoint-initdb.d container_name: nldi-liquibase diff --git a/liquibase/Dockerfile b/liquibase/Dockerfile index acafc0b..28338bb 100644 --- a/liquibase/Dockerfile +++ b/liquibase/Dockerfile @@ -1,29 +1,16 @@ -FROM usgswma/openjdk:debian-stretch-openjdk-11 +ARG LIQUIBASE_DOCKER_VERSION=3.10 -LABEL maintainer="[gs-w_eto_eb_federal_employees@usgs.gov](mailto:gs-w_eto_eb_federal_employees@usgs.gov)" +FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION} -############################################ -# Install Liquibase -############################################ +LABEL maintainer="[gs-w_eto_eb_federal_employees@usgs.gov](mailto:gs-w_eto_eb_federal_employees@usgs.gov)" -ARG LIQUIBASE_VERSION -ARG A_JDBC_JAR -ENV JDBC_JAR=$A_JDBC_JAR -ENV LIQUIBASE_HOME $HOME +ENV LIQUIBASE_HOME /liquibase ENV LIQUIBASE_WORKSPACE $LIQUIBASE_HOME/workspace ENV LOCALONLY "-c listen_addresses='127.0.0.1, ::1'" -ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/wait-for-it.sh - -ADD https://github.com/liquibase/liquibase/releases/download/liquibase-parent-$LIQUIBASE_VERSION/liquibase-$LIQUIBASE_VERSION-bin.tar.gz $LIQUIBASE_HOME/ - -ADD https://jdbc.postgresql.org/download/$JDBC_JAR $LIQUIBASE_HOME/lib/ - -RUN tar -xzf $LIQUIBASE_HOME/liquibase-$LIQUIBASE_VERSION-bin.tar.gz -C $LIQUIBASE_HOME/ - -RUN chmod -R 777 $LIQUIBASE_HOME - +USER root COPY ./docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh +USER liquibase + ENTRYPOINT ["docker-entrypoint.sh"]