diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 0a12ee0..efca327 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,7 +1,7 @@ { $schema: "https://docs.renovatebot.com/renovate-schema.json", extends: [ - "github>HSLdevcom/jore4-tools//renovatebot/jore4-default-preset.json5", + "github>HSLdevcom/jore4-tools/renovatebot/jore4-default-preset.json5", ], // repository-specific additional rules about grouping, exclusions and inclusions diff --git a/azuredbmock/00-initialize.sql b/azuredbmock/00-initialize.sql deleted file mode 100644 index 06eeaea..0000000 --- a/azuredbmock/00-initialize.sql +++ /dev/null @@ -1,73 +0,0 @@ - --- Initializations, which are needed locally, but not in the cloud / prod environments, --- go here. - --- These users are created from the jore4-deploy repository in cloud environments. -CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx'; -CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx'; -CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx'; -CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx'; -CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx'; - --- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html --- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any --- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the --- extensions from another schema. -CREATE EXTENSION IF NOT EXISTS pgcrypto; -CREATE EXTENSION IF NOT EXISTS postgis; -CREATE EXTENSION IF NOT EXISTS btree_gist; - --- allow hasura to create new schemas -GRANT CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx; - --- create database for auth and give ALL privileges to auth db user -CREATE DATABASE xxx_db_auth_name_xxx; -GRANT ALL ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx; - --- Make hasura role a member of jore3importer role because both roles must have --- ownership of tables and sequences since both are responsible for populating --- and truncating tables. In particular, sequence reset requires an ownership --- and cannot be granted as a privilege. -GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx; - --- create database for jore3 importer and give ALL privileges to jore3importer db user -CREATE DATABASE xxx_db_jore3importer_name_xxx; -GRANT ALL ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx; - --- create database for timetables and allow hasura to create new schemas in it -CREATE DATABASE xxx_db_timetables_name_xxx; -GRANT CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx; - --- create database for stop registry and give ALL privileges to Tiamat in it -CREATE DATABASE xxx_db_tiamat_name_xxx; -GRANT ALL ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_tiamat_username_xxx; - --- switch database context to timetables db to be able to add extensions there -\connect xxx_db_timetables_name_xxx; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -CREATE EXTENSION IF NOT EXISTS btree_gist; - --- interval outputs by default are using the sql format ('3 4:05:06'). Here we are switching to ISO 8601 format ('P3DT4H5M6S') -ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601'; - --- switch database context to stop db to initialize it to the state where tiamat can use it -\connect xxx_db_tiamat_name_xxx; -CREATE SCHEMA topology; -ALTER SCHEMA topology OWNER TO xxx_db_tiamat_username_xxx; -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; -CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; --- the postgis_topology creates two tables -ALTER TABLE topology.layer OWNER TO xxx_db_tiamat_username_xxx; -ALTER TABLE topology.topology OWNER TO xxx_db_tiamat_username_xxx; - --- grant hasura user read permissions to the tiamat database -GRANT CONNECT ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_hasura_username_xxx; - -GRANT USAGE ON SCHEMA public TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA public GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; - -GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; -GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; -ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/Dockerfile b/azuredbmock/Dockerfile index 8ad3520..fc1a18e 100644 --- a/azuredbmock/Dockerfile +++ b/azuredbmock/Dockerfile @@ -1,25 +1,22 @@ -# Builder docker image. -FROM postgis/postgis:12-3.1 +# As of 2025-02, in Azure PostgreSQL Flexible Server v15, the most recent +# available version of the PostGIS extension is 3.3.1. +FROM postgis/postgis:15-3.3 # fix collations to use fi_FI RUN localedef -i fi_FI -c -f UTF-8 -A /usr/share/locale/locale.alias fi_FI.UTF-8 ENV LANG fi_FI.utf8 -# install curl -RUN apt-get update && \ - apt-get install curl -y - # download script for reading docker secrets -RUN curl -o /tmp/read-secrets.sh "https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh" +ADD https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/read-secrets.sh /jore4/scripts/read-secrets.sh -COPY 00-initialize.sql /migrations/00-initialize.sql -COPY replace-placeholders-in-sql-schema-migrations.sh /scripts/replace-placeholders-in-sql-schema-migrations.sh +COPY migrations/ /jore4/migrations/ +COPY replace-placeholders-in-sql-schema-migrations.sh /jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh -COPY entrypoint.sh /usr/local/bin/ +COPY docker-entrypoint.sh /jore4/scripts/ -ENTRYPOINT ["entrypoint.sh"] +ENTRYPOINT ["/jore4/scripts/docker-entrypoint.sh"] CMD ["postgres"] # have to source the secrets here as well, otherwise the POSTGRES_USER and/or POSTGRES_DB variables might not exist HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=20 \ - CMD bash -c "source /tmp/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}" + CMD bash -c "source /jore4/scripts/read-secrets.sh && pg_isready -h localhost -U \${POSTGRES_USER} -d \${POSTGRES_DB}" diff --git a/azuredbmock/entrypoint.sh b/azuredbmock/docker-entrypoint.sh similarity index 79% rename from azuredbmock/entrypoint.sh rename to azuredbmock/docker-entrypoint.sh index 8ac606f..c0cdd9b 100755 --- a/azuredbmock/entrypoint.sh +++ b/azuredbmock/docker-entrypoint.sh @@ -3,16 +3,16 @@ set -Eeuo pipefail # read the secrets to environment variables -source /tmp/read-secrets.sh +source /jore4/scripts/read-secrets.sh # copy the migrations scripts to the docker-entrypoint-initdb.d folder so that the postgres entrypoint # executes them (together with the other migrations that might have been mapped as a volume) mkdir -p /docker-entrypoint-initdb.d -cp /migrations/* /docker-entrypoint-initdb.d/ +cp /jore4/migrations/* /docker-entrypoint-initdb.d/ # replace placeholders with secrets within the all migration SQL scripts SECRET_STORE_BASE_PATH="${SECRET_STORE_BASE_PATH:-/run/secrets}" -REPLACE_PLACEHOLDERS_SCRIPT='/scripts/replace-placeholders-in-sql-schema-migrations.sh' +REPLACE_PLACEHOLDERS_SCRIPT='/jore4/scripts/replace-placeholders-in-sql-schema-migrations.sh' MIGRATIONS_DIR="/docker-entrypoint-initdb.d/" "${REPLACE_PLACEHOLDERS_SCRIPT}" "${SECRET_STORE_BASE_PATH}" "${MIGRATIONS_DIR}" diff --git a/azuredbmock/migrations/01-create-roles.sql b/azuredbmock/migrations/01-create-roles.sql new file mode 100644 index 0000000..4de5033 --- /dev/null +++ b/azuredbmock/migrations/01-create-roles.sql @@ -0,0 +1,14 @@ +-- These database roles are also created in the azure-infra-jore4aks (Azure +-- DevOps) repository. +CREATE USER xxx_db_auth_username_xxx PASSWORD 'xxx_db_auth_password_xxx'; +CREATE USER xxx_db_jore3importer_username_xxx PASSWORD 'xxx_db_jore3importer_password_xxx'; +CREATE USER xxx_db_hasura_username_xxx PASSWORD 'xxx_db_hasura_password_xxx'; +CREATE USER xxx_db_tiamat_username_xxx PASSWORD 'xxx_db_tiamat_password_xxx'; +CREATE USER xxx_db_timetables_api_username_xxx PASSWORD 'xxx_db_timetables_api_password_xxx'; + +-- Make the hasura role a member of jore3importer role because both roles must +-- have ownership of tables and sequences in the default database (network and +-- routes) since both are responsible for populating and truncating tables in +-- the aforementioned database. In particular, sequence reset requires an +-- ownership and cannot be granted as a privilege. +GRANT xxx_db_jore3importer_username_xxx TO xxx_db_hasura_username_xxx; diff --git a/azuredbmock/migrations/02-create-network-database.sql b/azuredbmock/migrations/02-create-network-database.sql new file mode 100644 index 0000000..f1f3061 --- /dev/null +++ b/azuredbmock/migrations/02-create-network-database.sql @@ -0,0 +1,33 @@ +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions used, see https://hasura.io/docs/latest/graphql/core/deployment/postgres-requirements.html +-- Create the extensions in the public schema, since we'd need to give additional privileges ("use schema") to any +-- user who wishes to use these in the future. Also, Hasura would require additional setup to be able to use the +-- extensions from another schema. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_hasura_username_xxx; + +-- Grant required privileges in the public schema to the Hasura user. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to the Hasura +-- user. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the JORE3-Importer role to connect to the network database. +GRANT CONNECT ON DATABASE xxx_db_hasura_name_xxx TO xxx_db_jore3importer_username_xxx; + +-- Grant the JORE3-Importer role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/03-create-auth-database.sql b/azuredbmock/migrations/03-create-auth-database.sql new file mode 100644 index 0000000..07c86f0 --- /dev/null +++ b/azuredbmock/migrations/03-create-auth-database.sql @@ -0,0 +1,12 @@ +CREATE DATABASE xxx_db_auth_name_xxx; + +-- Allow the auth role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_auth_name_xxx TO xxx_db_auth_username_xxx; + +\connect xxx_db_auth_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Grant full schema access to the public schema to the auth role. +GRANT ALL ON SCHEMA public TO xxx_db_auth_username_xxx; diff --git a/azuredbmock/migrations/04-create-jore3importer-database.sql b/azuredbmock/migrations/04-create-jore3importer-database.sql new file mode 100644 index 0000000..40f7ad6 --- /dev/null +++ b/azuredbmock/migrations/04-create-jore3importer-database.sql @@ -0,0 +1,25 @@ +CREATE DATABASE xxx_db_jore3importer_name_xxx; + +-- Allow the jore3importer role to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_jore3importer_name_xxx TO xxx_db_jore3importer_username_xxx; + +\connect xxx_db_jore3importer_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +-- Create the extensions that JORE3-Importer needs. In PostgreSQL v15 server, +-- an ordinary user (without admin roles) may not be able to create extensions. +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant privileges in the public schema to the jore3importer role. +GRANT USAGE ON SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_jore3importer_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_jore3importer_username_xxx; + +-- Grant permission on the pg_catalog schema to conditionally create an +-- extension if the extension is not already created. This must be granted so +-- that the JORE3-Importer role can conditionally create the extension depending +-- on which database is involved (there are several in the importer's test setup). +GRANT SELECT ON TABLE pg_catalog.pg_extension TO xxx_db_jore3importer_username_xxx; diff --git a/azuredbmock/migrations/05-create-timetables-database.sql b/azuredbmock/migrations/05-create-timetables-database.sql new file mode 100644 index 0000000..8648b48 --- /dev/null +++ b/azuredbmock/migrations/05-create-timetables-database.sql @@ -0,0 +1,36 @@ +CREATE DATABASE xxx_db_timetables_name_xxx; + +-- Allow Hasura to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_hasura_username_xxx; + +-- Interval outputs by default are using the sql format ('3 4:05:06'). Here we +-- are switching to ISO 8601 format ('P3DT4H5M6S'). +ALTER DATABASE xxx_db_timetables_name_xxx SET intervalstyle = 'iso_8601'; + +-- Switch database context to be able to add extensions there. +\connect xxx_db_timetables_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + +-- Grant required privileges in the public schema to Hasura. +GRANT ALL ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_hasura_username_xxx; + +-- Grant select permissions on information_schema and pg_catalog to Hasura. +GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO xxx_db_hasura_username_xxx; + +-- Allow the timetables-api role to connect to the timetables database. +GRANT CONNECT ON DATABASE xxx_db_timetables_name_xxx TO xxx_db_timetables_api_username_xxx; + +-- Grant the timetables-api role to access the public schema and the objects and +-- functions created by extensions. Other schema-specific privileges are granted +-- in Hasura migrations. +GRANT USAGE ON SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_timetables_api_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_timetables_api_username_xxx; diff --git a/azuredbmock/migrations/06-create-stopregistry-database.sql b/azuredbmock/migrations/06-create-stopregistry-database.sql new file mode 100644 index 0000000..6ea599f --- /dev/null +++ b/azuredbmock/migrations/06-create-stopregistry-database.sql @@ -0,0 +1,39 @@ +CREATE DATABASE xxx_db_tiamat_name_xxx; + +-- Allow Tiamat to connect and create new schemas. +GRANT CONNECT, CREATE ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_tiamat_username_xxx; + +-- Switch database context to initialise it to the state where Tiamat can use +-- it. +\connect xxx_db_tiamat_name_xxx; + +-- Make the JORE4 admin role the owner of the public schema. +ALTER SCHEMA public OWNER TO CURRENT_USER; + +CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; + +-- Grant required privileges in the public schema to Tiamat. +GRANT ALL ON SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_tiamat_username_xxx; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO xxx_db_tiamat_username_xxx; + +-- Create "topology" schema and install the "postgis_topology" extension to it. +-- The Tiamat role needs ownership to the schema and its tables. +CREATE SCHEMA IF NOT EXISTS topology; +CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; +ALTER SCHEMA topology OWNER TO xxx_db_tiamat_username_xxx; +-- The "postgis_topology" extension creates two tables. +ALTER TABLE topology.layer OWNER TO xxx_db_tiamat_username_xxx; +ALTER TABLE topology.topology OWNER TO xxx_db_tiamat_username_xxx; + +-- Grant Hasura read permissions to the stop registry database. +GRANT CONNECT ON DATABASE xxx_db_tiamat_name_xxx TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA public TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA public GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx; + +GRANT USAGE ON SCHEMA topology TO xxx_db_hasura_username_xxx; +GRANT SELECT ON ALL TABLES IN SCHEMA topology TO xxx_db_hasura_username_xxx; +ALTER DEFAULT PRIVILEGES FOR USER xxx_db_tiamat_username_xxx IN SCHEMA topology GRANT SELECT ON TABLES TO xxx_db_hasura_username_xxx;