Add mainstem lookup table

database/ci/Dockerfile

@@ -10,6 +10,7 @@ LABEL org.opencontainers.image.authors="gs-w_eto_eb_federal_employees@usgs.gov"
 LABEL org.opencontainers.image.title="NLDI CI Database"
 LABEL org.opencontainers.image.description="A sample NLDI database used for continuous integration testing."
 
+ENV CONTEXTS=ci
 ENV LIQUIBASE_HOME /liquibase
 ENV LIQUIBASE_WORKSPACE /liquibase
 COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE}
@@ -24,4 +25,3 @@ RUN apt-get update && apt-get install -y curl
 COPY ./liquibase/changeLogs $LIQUIBASE_WORKSPACE
 COPY ./liquibase/scripts/*.sh /docker-entrypoint-initdb.d/
 COPY ./liquibase/scripts/dbInit /docker-entrypoint-initdb.d/
-COPY ./liquibase/scripts/dbCi /docker-entrypoint-initdb.d/

database/demo/Dockerfile

@@ -10,6 +10,7 @@ LABEL org.opencontainers.image.authors="gs-w_eto_eb_federal_employees@usgs.gov"
 LABEL org.opencontainers.image.title="NLDI Demo Database"
 LABEL org.opencontainers.image.description="A demo of the NLDI database containing sample data."
 
+ENV CONTEXTS=demo
 ENV LIQUIBASE_HOME /liquibase
 ENV LIQUIBASE_WORKSPACE /liquibase
 COPY --from=Liquibase ${LIQUIBASE_WORKSPACE} ${LIQUIBASE_WORKSPACE}
@@ -32,3 +33,8 @@ RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/downl
 RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_wqp_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_wqp_yahara.backup.gz
 RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_huc12pp_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_huc12pp_yahara.backup.gz
 RUN curl -L --verbose "https://github.com/internetofwater/nldi-db/releases/download/artifacts-1.0.0/feature_np21_nwis_yahara.backup.gz" -o $LIQUIBASE_HOME/feature_np21_nwis_yahara.backup.gz
+
+ARG MAINSTEM_COMMIT="9052cbaeccbaa88f91317f8ec9891a90d378371f"
+RUN curl -L "https://code.usgs.gov/wma/nhgf/gfv2.0/-/raw/${MAINSTEM_COMMIT}/workspace/data/mainstem_lookup.csv.gz?inline=false" \
+	-o ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz
+RUN gunzip ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz

liquibase/Dockerfile

@@ -22,11 +22,20 @@ ENV NLDI_READ_ONLY_PASSWORD=missing_read_only_password
 ENV POSTGRES_PASSWORD=missing_postgres_password
 
 USER root
+RUN mkdir -p ${LIQUIBASE_WORKSPACE}
+
+RUN apt-get --allow-releaseinfo-change update && apt-get install -y curl
 COPY ./docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
 USER liquibase
 
-COPY ./changeLogs /liquibase/workspace/.
+COPY ./changeLogs ${LIQUIBASE_WORKSPACE}/.
 COPY ./scripts/dbInit/* /docker-entrypoint-initdb.d/
 
+ARG MAINSTEM_COMMIT="9052cbaeccbaa88f91317f8ec9891a90d378371f"
+RUN curl -L "https://code.usgs.gov/wma/nhgf/gfv2.0/-/raw/${MAINSTEM_COMMIT}/workspace/data/mainstem_lookup.csv.gz?inline=false" \
+	-o ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz
+RUN gunzip ${LIQUIBASE_HOME}/mainstem_lookup.csv.gz
+
 ENTRYPOINT ["docker-entrypoint.sh"]

liquibase/changeLogs/nldi/nldi_data/changeLog.xml

@@ -15,5 +15,7 @@
 	<include file="functions/changeLog.xml" relativeToChangelogFile="true"/>
 	<include file="grants.sql" relativeToChangelogFile="true"/>
 	<include file="update_crawler_source/changeLog.yml" relativeToChangelogFile="true"/>
+	<include file="copyMainstemData.yml" relativeToChangelogFile="true"/>
+
 </databaseChangeLog>
 

liquibase/changeLogs/nldi/nldi_data/copyMainstemData.yml

@@ -0,0 +1,37 @@
+databaseChangeLog:
+  - preConditions:
+      - dbms:
+          type: PostgreSQL
+      - runningAs:
+          username: ${NLDI_DB_OWNER_USERNAME}
+
+  - changeSet:
+      author: egrahn
+      id: "load.nldi_data.mainstem_lookup"
+      runOnChange: true
+      context: "!ci"
+      changes:
+        - delete:
+            schemaName: nldi_data
+            tableName: mainstem_lookup
+        - loadData:
+            encoding: UTF-8
+            file: "${LIQUIBASE_HOME}/mainstem_lookup.csv"
+            relativeToChangelogFile: false
+            separator: ","
+            schemaName: nldi_data
+            tableName: mainstem_lookup
+            columns:
+                - column:
+                    index: 0
+                    name: nhdpv2_comid
+                    type: NUMERIC
+                - column:
+                    index: 1
+                    name: mainstem_id
+                    type: NUMERIC
+                - column:
+                    index: 2
+                    name: uri
+                    type: STRING
+

liquibase/changeLogs/nldi/nldi_data/tables.sql

@@ -102,6 +102,11 @@ drop table nldi_data.feature_np21_nwis_temp;
 --precondition-sql-check expectedResult:t select to_regclass('nldi_data."feature; select * from pg_class;_temp"') is not null
 drop table nldi_data."feature; select * from pg_class;_temp";
 
+--changeset egrahn:drop.nldi_data.sqlinjection_test
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:t select to_regclass('nldi_data."feature; select * from pg_class;"') is not null
+drop table nldi_data."feature; select * from pg_class;";
+
 --changeset egrahn:drop.nldi_data.feature_huc12pp_temp
 --preconditions onFail:MARK_RAN onError:HALT
 --precondition-sql-check expectedResult:t select to_regclass('nldi_data.feature_huc12pp_temp') is not null
@@ -112,3 +117,15 @@ drop table nldi_data.feature_huc12pp_temp;
 --precondition-sql-check expectedResult:0 select count(column_name) from information_schema."columns" where table_schema = 'nldi_data' and table_name = 'feature' and column_name = 'shape'
 alter table nldi_data.feature add column shape geometry(geometry,4269);
 --rollback alter table nldi_data.feature drop column if exists shape;
+
+--changeset egrahn:create.nldi_data.mainstem_lookup
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'mainstem_lookup'
+create table nldi_data.mainstem_lookup
+(
+  nhdpv2_comid integer,
+  mainstem_id integer,
+  uri text
+);
+alter table nldi_data.mainstem_lookup owner to ${NLDI_SCHEMA_OWNER_USERNAME};
+--rollback drop table nldi_data.mainstem_lookup;

liquibase/changeLogs/nldi/nldi_data/update_crawler_source/changeLog.yml

@@ -9,6 +9,7 @@ databaseChangeLog:
       author: kkehl
       id: "load.nldi_data.update_crawler_source"
       runOnChange: true
+      context: "!ci"
       changes:
         - delete:
             schemaName: nldi_data

liquibase/scripts/dbInit/z1_postgres_liquibase.sh

@@ -1,11 +1,5 @@
 #!/bin/bash 
 
-# create the nldi project user and database
-#psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
-#	create role nldi with login createrole password '${NLDI_PASSWORD}';
-#	alter database nldi owner to nldi;
-#EOSQL
-
 # postgres to postgres db scripts
 ${LIQUIBASE_HOME}/liquibase \
 	--classpath=${LIQUIBASE_HOME}/lib/${JDBC_JAR} \

liquibase/scripts/dbInit/z2_nldi_liquibase.sh

@@ -15,4 +15,4 @@ ${LIQUIBASE_HOME}/liquibase \
 	-DNLDI_DB_OWNER_USERNAME=${NLDI_DB_OWNER_USERNAME} \
 	-DNLDI_SCHEMA_OWNER_USERNAME=${NLDI_SCHEMA_OWNER_USERNAME} \
 	-DNHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME} \
-	-DNLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME}
+	-DNLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME}

liquibase/scripts/z0_restart.sh

@@ -1,4 +1,4 @@
 #!/bin/bash 
 
 # Restart postgres to make sure we can connect
-pg_ctl -D "$PGDATA" -m fast -o "$LOCALONLY" -w restart
+pg_ctl -D "$PGDATA" -m fast -o "$LOCALONLY" -w restart