diff --git a/database/ci/Dockerfile b/database/ci/Dockerfile index aab95e2..8312292 100755 --- a/database/ci/Dockerfile +++ b/database/ci/Dockerfile @@ -10,6 +10,7 @@ LABEL org.opencontainers.image.authors="gs-w_eto_eb_federal_employees@usgs.gov" LABEL org.opencontainers.image.title="NLDI CI Database" LABEL org.opencontainers.image.description="A sample NLDI database used for continuous integration testing." +ENV CONTEXTS=ci ENV LIQUIBASE_HOME /liquibase ENV LIQUIBASE_WORKSPACE /liquibase COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE} @@ -24,4 +25,3 @@ RUN apt-get update && apt-get install -y curl COPY ./liquibase/changeLogs $LIQUIBASE_WORKSPACE COPY ./liquibase/scripts/*.sh /docker-entrypoint-initdb.d/ COPY ./liquibase/scripts/dbInit /docker-entrypoint-initdb.d/ -COPY ./liquibase/scripts/dbCi /docker-entrypoint-initdb.d/ diff --git a/database/demo/Dockerfile b/database/demo/Dockerfile index f435cfe..1cf42f3 100755 --- a/database/demo/Dockerfile +++ b/database/demo/Dockerfile @@ -10,6 +10,7 @@ LABEL org.opencontainers.image.authors="gs-w_eto_eb_federal_employees@usgs.gov" LABEL org.opencontainers.image.title="NLDI Demo Database" LABEL org.opencontainers.image.description="A demo of the NLDI database containing sample data." +ENV CONTEXTS=demo ENV LIQUIBASE_HOME /liquibase ENV LIQUIBASE_WORKSPACE /liquibase COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE} @@ -32,3 +33,8 @@ RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/downl RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_wqp_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_wqp_yahara.backup.gz RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_huc12pp_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_huc12pp_yahara.backup.gz RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_np21_nwis_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_np21_nwis_yahara.backup.gz + +ARG MAINSTEM_COMMIT="9052cbaeccbaa88f91317f8ec9891a90d378371f" +RUN curl -L "https://code.usgs.gov/wma/nhgf/gfv2.0/-/raw/${MAINSTEM_COMMIT}/workspace/data/mainstem_lookup.csv.gz?inline=false" \ + -o ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz +RUN gunzip ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz diff --git a/liquibase/Dockerfile b/liquibase/Dockerfile index f72bf9a..929260f 100644 --- a/liquibase/Dockerfile +++ b/liquibase/Dockerfile @@ -22,11 +22,20 @@ ENV NLDI_READ_ONLY_PASSWORD=missing_read_only_password ENV POSTGRES_PASSWORD=missing_postgres_password USER root +RUN mkdir -p ${LIQUIBASE_WORKSPACE} + +RUN apt-get --allow-releaseinfo-change update && apt-get install -y curl COPY ./docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh + USER liquibase -COPY ./changeLogs /liquibase/workspace/. +COPY ./changeLogs ${LIQUIBASE_WORKSPACE}/. COPY ./scripts/dbInit/* /docker-entrypoint-initdb.d/ +ARG MAINSTEM_COMMIT="9052cbaeccbaa88f91317f8ec9891a90d378371f" +RUN curl -L "https://code.usgs.gov/wma/nhgf/gfv2.0/-/raw/${MAINSTEM_COMMIT}/workspace/data/mainstem_lookup.csv.gz?inline=false" \ + -o ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz +RUN gunzip ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz + ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/liquibase/changeLogs/nldi/nldi_data/changeLog.xml b/liquibase/changeLogs/nldi/nldi_data/changeLog.xml index e963624..91a1bd2 100755 --- a/liquibase/changeLogs/nldi/nldi_data/changeLog.xml +++ b/liquibase/changeLogs/nldi/nldi_data/changeLog.xml @@ -15,5 +15,7 @@ + + diff --git a/liquibase/changeLogs/nldi/nldi_data/copyMainstemData.yml b/liquibase/changeLogs/nldi/nldi_data/copyMainstemData.yml new file mode 100644 index 0000000..2c150ab --- /dev/null +++ b/liquibase/changeLogs/nldi/nldi_data/copyMainstemData.yml @@ -0,0 +1,37 @@ +databaseChangeLog: + - preConditions: + - dbms: + type: PostgreSQL + - runningAs: + username: ${NLDI_DB_OWNER_USERNAME} + + - changeSet: + author: egrahn + id: "load.nldi_data.mainstem_lookup" + runOnChange: true + context: "!ci" + changes: + - delete: + schemaName: nldi_data + tableName: mainstem_lookup + - loadData: + encoding: UTF-8 + file: "${LIQUIBASE_HOME}/mainstem_lookup.csv" + relativeToChangelogFile: false + separator: "," + schemaName: nldi_data + tableName: mainstem_lookup + columns: + - column: + index: 0 + name: nhdpv2_comid + type: NUMERIC + - column: + index: 1 + name: mainstem_id + type: NUMERIC + - column: + index: 2 + name: uri + type: STRING + \ No newline at end of file diff --git a/liquibase/changeLogs/nldi/nldi_data/tables.sql b/liquibase/changeLogs/nldi/nldi_data/tables.sql index e47f7f8..7319bf1 100755 --- a/liquibase/changeLogs/nldi/nldi_data/tables.sql +++ b/liquibase/changeLogs/nldi/nldi_data/tables.sql @@ -102,6 +102,11 @@ drop table nldi_data.feature_np21_nwis_temp; --precondition-sql-check expectedResult:t select to_regclass('nldi_data."feature; select * from pg_class;_temp"') is not null drop table nldi_data."feature; select * from pg_class;_temp"; +--changeset egrahn:drop.nldi_data.sqlinjection_test +--preconditions onFail:MARK_RAN onError:HALT +--precondition-sql-check expectedResult:t select to_regclass('nldi_data."feature; select * from pg_class;"') is not null +drop table nldi_data."feature; select * from pg_class;"; + --changeset egrahn:drop.nldi_data.feature_huc12pp_temp --preconditions onFail:MARK_RAN onError:HALT --precondition-sql-check expectedResult:t select to_regclass('nldi_data.feature_huc12pp_temp') is not null @@ -112,3 +117,15 @@ drop table nldi_data.feature_huc12pp_temp; --precondition-sql-check expectedResult:0 select count(column_name) from information_schema."columns" where table_schema = 'nldi_data' and table_name = 'feature' and column_name = 'shape' alter table nldi_data.feature add column shape geometry(geometry,4269); --rollback alter table nldi_data.feature drop column if exists shape; + +--changeset egrahn:create.nldi_data.mainstem_lookup +--preconditions onFail:MARK_RAN onError:HALT +--precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'mainstem_lookup' +create table nldi_data.mainstem_lookup +( + nhdpv2_comid integer, + mainstem_id integer, + uri text +); +alter table nldi_data.mainstem_lookup owner to ${NLDI_SCHEMA_OWNER_USERNAME}; +--rollback drop table nldi_data.mainstem_lookup; diff --git a/liquibase/changeLogs/nldi/nldi_data/update_crawler_source/changeLog.yml b/liquibase/changeLogs/nldi/nldi_data/update_crawler_source/changeLog.yml index b66cb48..4954e21 100644 --- a/liquibase/changeLogs/nldi/nldi_data/update_crawler_source/changeLog.yml +++ b/liquibase/changeLogs/nldi/nldi_data/update_crawler_source/changeLog.yml @@ -9,6 +9,7 @@ databaseChangeLog: author: kkehl id: "load.nldi_data.update_crawler_source" runOnChange: true + context: "!ci" changes: - delete: schemaName: nldi_data diff --git a/liquibase/scripts/dbCi/z3_clear_data.sh b/liquibase/scripts/dbCi/z3_clear_data.sh deleted file mode 100755 index 98ac3f1..0000000 --- a/liquibase/scripts/dbCi/z3_clear_data.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# The CI database should be completely cleared of data. -# The crawler_source table has data injected from the liquibase -# changelogs, so it needs to be truncated. -psql --host=127.0.0.1 \ - --port=5432 \ - --username=postgres \ - --no-password \ - --dbname=${NLDI_DATABASE_NAME} \ - --command='truncate table nldi_data.crawler_source;' - -exit 0 diff --git a/liquibase/scripts/dbInit/z1_postgres_liquibase.sh b/liquibase/scripts/dbInit/z1_postgres_liquibase.sh index f4cd062..d9cca64 100755 --- a/liquibase/scripts/dbInit/z1_postgres_liquibase.sh +++ b/liquibase/scripts/dbInit/z1_postgres_liquibase.sh @@ -1,11 +1,5 @@ #!/bin/bash -# create the nldi project user and database -#psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL -# create role nldi with login createrole password '${NLDI_PASSWORD}'; -# alter database nldi owner to nldi; -#EOSQL - # postgres to postgres db scripts ${LIQUIBASE_HOME}/liquibase \ --classpath=${LIQUIBASE_HOME}/lib/${JDBC_JAR} \ diff --git a/liquibase/scripts/dbInit/z2_nldi_liquibase.sh b/liquibase/scripts/dbInit/z2_nldi_liquibase.sh index ba2b40e..91f1d61 100644 --- a/liquibase/scripts/dbInit/z2_nldi_liquibase.sh +++ b/liquibase/scripts/dbInit/z2_nldi_liquibase.sh @@ -15,4 +15,4 @@ ${LIQUIBASE_HOME}/liquibase \ -DNLDI_DB_OWNER_USERNAME=${NLDI_DB_OWNER_USERNAME} \ -DNLDI_SCHEMA_OWNER_USERNAME=${NLDI_SCHEMA_OWNER_USERNAME} \ -DNHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME} \ - -DNLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME} \ No newline at end of file + -DNLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME} diff --git a/liquibase/scripts/z0_restart.sh b/liquibase/scripts/z0_restart.sh index 5a78518..e3b9c6c 100644 --- a/liquibase/scripts/z0_restart.sh +++ b/liquibase/scripts/z0_restart.sh @@ -1,4 +1,4 @@ #!/bin/bash # Restart postgres to make sure we can connect -pg_ctl -D "$PGDATA" -m fast -o "$LOCALONLY" -w restart \ No newline at end of file +pg_ctl -D "$PGDATA" -m fast -o "$LOCALONLY" -w restart