update infraLinks.sql to Digiroad 2025-02 routes and move to Azure

.github/workflows/cd.yml

@@ -77,11 +77,12 @@ jobs:
     needs:
       - publish_ui_docker_image_to_acr
       - publish_cypress_docker_image_to_acr
-    uses: HSLdevcom/jore4-tools/.github/workflows/shared-run-e2e.yml@main
+    uses: HSLdevcom/jore4-tools/.github/workflows/shared-run-e2e.yml@seed-infrastructure-links-v4
     with:
       ui_version: ${{ needs.publish_ui_docker_image_to_acr.outputs.docker_image }}
       cypress_version: ${{ needs.publish_cypress_docker_image_to_acr.outputs.docker_image }}
       update_e2e_test_durations: ${{ github.ref == 'refs/heads/main' }}
       test-tags: ''
+      custom_docker_compose: docker/docker-compose.custom-e2e.yml
     secrets:
       jore4_ci_data_repo_ssh_key: ${{ secrets.JORE4_CI_DATA_REPO_DEPLOY_KEY }}

.github/workflows/check-generated-resources.yml

@@ -39,4 +39,10 @@ jobs:
         run: yarn ws:codegen generate
 
       - name: Verify that generate command made no changes, so schema is up to date
-        run: '[ -z "$(git status --porcelain)" ]'
+        run: |
+          STATUS=$(git status --porcelain)
+          echo "Git status output:"
+          echo "$STATUS"
+          echo "=== Git diff ==="
+          git diff
+          [ -z "$(git status --porcelain)" ]

.gitignore

@@ -13,6 +13,7 @@ yarn-error.log*
 docker/*
 !docker/docker-compose.cypress.yml
 !docker/docker-compose.custom.yml
+!docker/docker-compose.custom-e2e.yml
 !docker/docker-compose.testdb-volume.yml
 !docker/docker-compose.e2e.yml
 
@@ -24,3 +25,7 @@ tsconfig.tsbuildinfo
 
 dump.sql
 *.pgdump
+infraLinks.sql
+infra_links_data.sql
+
+.DS_Store

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "jore4-hasura"]
 	path = jore4-hasura
 	url = https://github.com/HSLdevcom/jore4-hasura
+	branch = digiroad_links

README.md

@@ -213,7 +213,7 @@ To update database dump files (with the `.pgdump` extension), do the following:
 
 ## Regenerating infraLinks.sql
 
-After updating dump file for the network & routes database, you may consider updating the seed data for infrastructure links which is located at `./test-db-manager/src/dumps/infraLinks/infraLinks.sql`.
+After updating dump file for the network & routes database, you may consider updating the seed data for infrastructure links which is located in azure blob storage stjore4dev001's jore4-dump container. Download with `scripts/development.sh infralink:download`, after which the file is at `./infraLinks.sql`.
 
 To do that:
 
@@ -226,6 +226,7 @@ To do that:
   COPY infrastructure_network.infrastructure_link (infrastructure_link_id, direction, shape, estimated_length_in_metres, external_link_id, external_link_source) FROM stdin;
   ```
 - Copy the command and the immediately following rows of data (over 150 000 rows in total) and replace the same command in the `infraLinks.sql` file to update infrastructure seed data. Make sure you only copy infrastructure link data!
+- upload the infraLinks.sql to azure (stjore4dev001's jore4-dump container) and update the location to development.sh (INFRALINKS_FILENAME constant).
 
 ### Fixing timetables seed data (NEEDS UPDATE)
 

cypress/datasets/base.ts

@@ -19,14 +19,14 @@ import { DateTime } from 'luxon';
 import { UUID } from '../types';
 
 export const testInfraLinkExternalIds = [
-  '442321', // on Annankatu, between Bulevardi and Lönnrotinkatu and
-  '442326', // on Annankatu, between Lönnrotinkatu and Kalevankatu
-  '442317', // on Kalevankatu, between Annankatu and Fredrikinkatu
-  '442027', // on Kalevankatu, between Fredrikinkatu and Albertinkatu
-  '442438', // on Albertinkatu, between Kalevankatu and Lönnrotinkatu (one-way)
-  '445113', // on Lönnrotinkatu, between Albertinkatu and Abrahaminkatu (one-way)
-  '445118', // on Abrahaminkatu, between Lönnrotinkatu and Kalevankatu
-  '442423', // on Kalevankatu, between Abrahaminkatu and Albertinkatu
+  '145c33b2-5d2a-4668-b922-6117408772e0:1', // on Annankatu, between Bulevardi and Lönnrotinkatu and
+  'f8af6fe3-b711-4365-aad9-54a59b26a8aa:1', // on Annankatu, between Lönnrotinkatu and Kalevankatu
+  '11b3a892-22a8-49b0-b2f7-0dae577f46ca:1', // on Kalevankatu, between Annankatu and Fredrikinkatu
+  'd9920aa3-748f-486b-8a44-7c07e55a9d08:1', // on Kalevankatu, between Fredrikinkatu and Albertinkatu
+  'f7953bf5-7bee-426b-a6cb-7b136236630b:1', // on Albertinkatu, between Kalevankatu and Lönnrotinkatu (one-way)
+  '87c2fbd0-697c-460a-ab38-8b0b41eeead1:1', // on Lönnrotinkatu, between Albertinkatu and Abrahaminkatu (one-way)
+  'a87caa98-2d9d-45fa-9dff-3909b7d7d763:1', // on Abrahaminkatu, between Lönnrotinkatu and Kalevankatu
+  '1b0dcad2-a363-482d-a630-5ef8cfcd0fec:1', // on Kalevankatu, between Abrahaminkatu and Albertinkatu
 ];
 
 const timingPlaces = [

cypress/e2e/stop-registry/stopDetails.cy.ts

@@ -53,15 +53,15 @@ import { InsertedStopRegistryIds } from '../utils';
 
 const testInfraLinks = [
   {
-    externalId: '445156',
+    externalId: '7d29bd61-6cf7-4d2c-8bd8-b8e835fe90b7:1',
     coordinates: [24.92669962, 60.16418108, 10.09699999],
   },
   {
-    externalId: '442424',
+    externalId: 'cbe70fa1-8797-4dd4-b264-5a69f2ddcfc9:1',
     coordinates: [24.92904198, 60.16490775, 0],
   },
   {
-    externalId: '442325',
+    externalId: 'c1e17eee-96d6-4d83-91d2-51512318bff2:1',
     coordinates: [24.93207242, 60.16600322, 0],
   },
 ];

docker/docker-compose.custom-e2e.yml

@@ -0,0 +1,4 @@
+services:
+  jore4-mapmatchingdb:
+    environment:
+      DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-29_create_routing_schema_digiroad_r_2025_02_fixup.sql"

docker/docker-compose.custom.yml

@@ -10,7 +10,7 @@ services:
     # The :hsl-tag contains the desired version of hsl specific hasura.
     # Waiting for merging feature-branch to main in hasura-repo
     # Note: also update jore4-hasura-e2e in docker-compose.e2e when changing this.
-    image: "crjore4prod001.azurecr.io/jore4-hsl-hasura:main-2025-11-03-a11bef7ade55dbe7c61a11e02bee65908e44febe"
+    image: 'crjore4prod001.azurecr.io/jore4-hsl-hasura:digiroad_links-2025-11-04-8cecf9f21042ed0d053d86aae10eda0dc92acdf6'
     # Waiting for database to be ready to avoid startup delay due to hasura crashing at startup if db is offline
     # Note: this should only be done in development setups as Kubernetes does not allow waiting for services to be ready
     depends_on:
@@ -44,3 +44,13 @@ services:
     # Pin timetables api to a compatible version.
     # Note: also update jore4-timetablesapi-e2e in docker-compose.e2e when changing this.
     image: 'hsldevcom/jore4-timetables-api:main--20240905-da293a99ddeff4776bc7c20d675832b0a407fe9f'
+
+  jore4-mapmatchingdb:
+    environment:
+      DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-29_create_routing_schema_digiroad_r_2025_02_fixup.sql"
+
+  jore4-hasura-e2e:
+    image: 'crjore4prod001.azurecr.io/jore4-hsl-hasura:digiroad_links-2025-11-04-8cecf9f21042ed0d053d86aae10eda0dc92acdf6'
+
+  jore4-hasura-base:
+    image: 'crjore4prod001.azurecr.io/jore4-hsl-hasura:digiroad_links-2025-11-04-8cecf9f21042ed0d053d86aae10eda0dc92acdf6'

docker/docker-compose.e2e.yml

@@ -43,7 +43,7 @@ services:
     extends:
       file: docker-compose.base.yml
       service: jore4-hasura-base
-    image: "crjore4prod001.azurecr.io/jore4-hsl-hasura:main-2025-11-03-a11bef7ade55dbe7c61a11e02bee65908e44febe"
+    image: "crjore4prod001.azurecr.io/jore4-hsl-hasura:digiroad_links-2025-11-04-8cecf9f21042ed0d053d86aae10eda0dc92acdf6"
     container_name: "hasura-e2e"
     ports:
       - "127.0.0.1:3211:8080"
@@ -58,3 +58,7 @@ services:
         condition: service_healthy
       jore4-tiamat-e2e:
         condition: service_healthy
+
+  jore4-mapmatchingdb:
+    environment:
+      DIGIROAD_ROUTING_DUMP_URL: "https://stjore4dev001.blob.core.windows.net/jore4-digiroad/2025-08-29_create_routing_schema_digiroad_r_2025_02_fixup.sql"

scripts/development.sh

@@ -16,9 +16,10 @@ DOCKER_COMPOSE_BUNDLE_REF=${BUNDLE_REF:-main}
 # this project from others.
 export COMPOSE_PROJECT_NAME=jore4-ui
 
-DUMP_ROUTES_FILENAME="2025-04-03_test/2025-04-03-jore4-local-jore4e2e.pgdump"
-DUMP_TIMETABLES_FILENAME="2025-04-03_test/2025-04-03-jore4-local-timetablesdb-nodata.pgdump"
-DUMP_STOPS_FILENAME="2025-04-03_test/2025-04-03-jore4-local-stopdb.pgdump"
+DUMP_ROUTES_FILENAME="2025-09-24_local_test/2025-09-24-jore4-local-jore4e2e.pgdump"
+DUMP_TIMETABLES_FILENAME="2025-09-24_local_test/2025-09-24-jore4-local-timetablesdb-nodata.pgdump"
+DUMP_STOPS_FILENAME="2025-09-24_local_test/2025-09-24-jore4-local-stopdb.pgdump"
+INFRALINKS_URL="https://stjore4dev001.blob.core.windows.net/jore4-ui/2025-09-24-infraLinks.sql"
 
 DOCKER_TESTDB_IMAGE="jore4-testdb"
 DOCKER_IMAGES=("jore4-auth" "jore4-hasura" "jore4-mbtiles" "jore4-mapmatchingdb" "jore4-mapmatching" "jore4-hastus" "jore4-tiamat" "jore4-timetablesapi")
@@ -71,11 +72,22 @@ wait_for_database() {
   done
 }
 
+download_infralinks() {
+  if [ -f "infraLinks.sql" ]; then
+    echo "infraLinks.sql already exists, skipping download."
+    return
+  fi
+  echo "Downloading infraLinks.sql..."
+  curl "$INFRALINKS_URL" -o "infraLinks.sql"
+}
+
 seed_infra_links() {
+  download_infralinks
+
   echo "$1: Seeding infrastructure links..."
 
   wait_for_database "$1" infrastructure_network infrastructure_link
-  docker exec -i "$1" psql $ROUTES_DB_CONNECTION_STRING < test-db-manager/src/dumps/infraLinks/infraLinks.sql;
+  docker exec -i "$1" psql $ROUTES_DB_CONNECTION_STRING < "infraLinks.sql";
 }
 
 check_pinned_image() {
@@ -236,6 +248,7 @@ download_dump() {
       --container-name "jore4-dump" \
       --name "$az_blob_filepath" \
       --file "$(basename "$az_blob_filepath")" \
+      --output none \
       --auth-mode login
   fi
 }
@@ -373,6 +386,10 @@ print_usage() {
     the BUNDLE_REF environment variable. By default, the latest version is
     downloaded.
 
+  infralinks:download
+    Downloads the infrastructure links seed data SQL file (infraLinks.sql) from Azure
+    Blob Storage.
+
   dump:download [<azure_blob_filepath>]
     Downloads a JORE4 database dump from Azure Blob Storage. A full file path
     may be given as a parameter. The file path is used to refer to a file inside
@@ -421,12 +438,16 @@ setup:test)
   setup_environment test
   ;;
 
+infralinks:download)
+  download_infralinks
+  ;;
+
 dump:download)
   download_dump
   ;;
 
 dump:import)
-  import_dump
+  import_dump "$2" "$3"
   ;;
 
 digitransit:fetch)

scripts/seed-infrastructure-links.sh

@@ -1,7 +1,20 @@
 #!/usr/bin/env bash
 
+if [ -z "$1" ]; then
+  echo "Usage: $0 <docker_container_name>"
+  exit 1
+fi
+
 cd "$(dirname "$0")"/..
 
+if [ ! -f infraLinks.sql ]; then
+  ./scripts/development.sh infralinks:download
+  if [ ! -f infraLinks.sql ]; then
+    echo "Failed to download infrastructure links data."
+    exit 1
+  fi
+fi
+
 DB_CONNECTION_STRING=postgresql://dbadmin:adminpassword@localhost:5432/jore4e2e
 
 echo "$1: Seeding infrastructure links..."
@@ -11,7 +24,7 @@ while ! $SUCCESS; do
   echo "$1: Checking if infrastructure link schema exists..."
   if [[ $(docker exec $1 psql $DB_CONNECTION_STRING -AXqtc "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = 'infrastructure_network' AND tablename = 'infrastructure_link');") = "t" ]]; then
     echo "$1: Schema found! Seeding infrastructure links..."
-      docker exec -i $1 psql $DB_CONNECTION_STRING < test-db-manager/src/dumps/infraLinks/infraLinks.sql;
+      docker exec -i $1 psql $DB_CONNECTION_STRING < infraLinks.sql;
       SUCCESS=true
   fi
   sleep 2