Seeding pipelines with par-stream

bbox-tile-server/Cargo.toml

@@ -32,28 +32,30 @@ futures-util = "0.3.21"
 geozero = { workspace = true, features = [ "with-mvt", "with-postgis-sqlx" ] }
 indicatif = "0.16.2"
 log = { workspace = true }
-martin-mbtiles = { package = "mbtiles", version = "0.7.2", default-features = false }
-martin-tile-utils = "0.1.3"
+martin-mbtiles = { package = "mbtiles", version = "0.9.1", default-features = false }
+martin-tile-utils = "0.4.1"
 num_cpus = { workspace = true }
 once_cell = { workspace = true }
 pmtiles = { version = "0.3.1", features = ["mmap-async-tokio"] }
 pmtiles2 = { version = "0.2.2", default-features = false }
 prometheus = { workspace = true }
 reqwest = { workspace = true }
-rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] }
-rusoto_s3 = { version = "0.48.0", default-features = false, features = ["rustls"] }
+rusoto_core = { version = "0.47.0", default-features = false, features = ["rustls"] }
+rusoto_s3 = { version = "0.47.0", default-features = false, features = ["rustls"] }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sqlx = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 #tile-grid = "0.5.2"
 tile-grid = { git = "https://github.com/pka/tile-grid" }
-tilejson = "0.3.2"
+tilejson = "0.4.1"
 tokio = { version = "1.17.0", features = ["rt-multi-thread", "fs", "sync"] }
 toml = "0.8.10"
 regex = "1.10.3"
 geo-types = "0.7.13"
+par-stream = { version = "0.10.2", features = ["runtime-tokio"] }
+flate2 = "1.0.28"
 
 [[bin]]
 name = "bbox-tile-server"

bbox-tile-server/bbox-mvtbench.toml

@@ -0,0 +1,127 @@
+[[datasource]]
+name = "pg"   
+[datasource.postgis]
+#url = "postgresql://mvtbench:mvtbench@mvtbenchdb/mvtbench"
+url = "postgresql://mvtbench:mvtbench@127.0.0.1:5439/mvtbench"
+
+[[tilestore]]
+name = "tilecache"    
+[tilestore.files]
+#base_dir = "../tiles"
+base_dir = "/tmp/mvtbench"
+
+[[tilestore]]
+name = "mbtilecache"
+[tilestore.mbtiles]
+path = "/tmp/tilecache.mbtiles"
+
+[[tilestore]]
+name = "pmtilecache"
+[tilestore.pmtiles]
+path = "/tmp/tilecache.pmtiles"
+
+[[tileset]]
+name = "ne_countries"
+tms = "WebMercatorQuad"
+cache = "tilecache"
+[tileset.postgis]
+minzoom = 0
+maxzoom = 6
+attribution = "Natural Earth v4"
+postgis2 = false
+[tileset.postgis.extent]
+minx = -179.97277
+miny = -83.05457
+maxx = 179.99366
+maxy = 83.23559
+
+[[tileset.postgis.layer]]
+name = "country"
+geometry_field = "wkb_geometry"
+geometry_type = "MULTIPOLYGON"
+srid = 3857
+no_transform = false
+tile_size = 4096
+simplify = true
+tolerance = "!pixel_width!/2"
+buffer_size = 3
+make_valid = false
+shift_longitude = false
+
+[[tileset.postgis.layer.query]]
+minzoom = 0
+sql = "SELECT wkb_geometry, adm0_a3, mapcolor7 FROM ne_10m_admin_0_countries WHERE min_zoom::integer <= !zoom! AND wkb_geometry && !bbox!"
+
+[[tileset.postgis.layer]]
+name = "country-name"
+geometry_field = "wkb_geometry"
+geometry_type = "POINT"
+srid = 3857
+no_transform = false
+tile_size = 4096
+simplify = false
+tolerance = "!pixel_width!/2"
+buffer_size = 0
+make_valid = false
+shift_longitude = false
+
+[[tileset.postgis.layer.query]]
+minzoom = 0
+sql = "SELECT wkb_geometry, abbrev, name FROM ne_10m_admin_0_country_points"
+
+[[tileset.postgis.layer]]
+name = "geo-lines"
+geometry_field = "wkb_geometry"
+geometry_type = "MULTILINESTRING"
+srid = 3857
+no_transform = false
+tile_size = 4096
+simplify = false
+tolerance = "!pixel_width!/2"
+buffer_size = 0
+make_valid = false
+shift_longitude = false
+
+[[tileset.postgis.layer.query]]
+minzoom = 1
+maxzoom = 4
+sql = "SELECT wkb_geometry, name FROM ne_50m_geographic_lines"
+
+[[tileset.postgis.layer.query]]
+minzoom = 5
+sql = "SELECT wkb_geometry, name FROM ne_10m_geographic_lines"
+
+[[tileset.postgis.layer]]
+name = "land-border-country"
+geometry_field = "wkb_geometry"
+geometry_type = "MULTILINESTRING"
+srid = 3857
+no_transform = false
+fid_field = "ogc_fid"
+tile_size = 4096
+simplify = true
+tolerance = "!pixel_width!/2"
+buffer_size = 0
+make_valid = false
+shift_longitude = false
+
+[[tileset.postgis.layer.query]]
+minzoom = 0
+sql = "SELECT wkb_geometry FROM ne_10m_admin_0_boundary_lines_land WHERE min_zoom::integer <= !zoom! AND wkb_geometry && !bbox!"
+
+[[tileset.postgis.layer]]
+name = "state"
+geometry_field = "wkb_geometry"
+geometry_type = "MULTILINESTRING"
+srid = 3857
+no_transform = false
+tile_size = 4096
+simplify = true
+tolerance = "!pixel_width!/2"
+buffer_size = 0
+make_valid = false
+shift_longitude = false
+
+[[tileset.postgis.layer.query]]
+minzoom = 0
+sql = "SELECT wkb_geometry, adm0_a3 FROM ne_10m_admin_1_states_provinces_lines WHERE min_zoom::integer <= !zoom! AND wkb_geometry && !bbox!"

bbox-tile-server/justfile

@@ -0,0 +1,127 @@
+#!/usr/bin/env just --justfile
+
+set shell := ["bash", "-c"]
+
+# Start test database
+start-db: start-docker-db docker-is-ready
+
+# Start test database
+[private]
+start-docker-db:
+    docker run -p 127.0.0.1:5439:5432 -d --name mvtbenchdb --rm sourcepole/mvtbenchdb
+
+# Stop test database
+stop-db:
+    docker stop mvtbenchdb
+
+# Wait for the test database to be ready
+[private]
+docker-is-ready:
+    docker exec mvtbenchdb sh -c 'until pg_isready -h localhost -U postgres; do sleep 1; done'
+
+# -- s3 test environment
+
+start-minio:
+    mkdir -p s3data
+    docker run --security-opt label=disable -d --rm --name minio -p 9000:9000 -p 9001:9001 -v $PWD/s3data:/data -e MINIO_REGION_NAME=my-region -e MINIO_ROOT_USER=miniostorage -e MINIO_ROOT_PASSWORD=miniostorage minio/minio server /data --console-address ":9001"
+
+stop-minio:
+    docker stop minio
+
+setup-minio:
+    docker exec minio mc config host add local-docker http://localhost:9000 miniostorage miniostorage
+    docker exec minio mc mb local-docker/tiles || true
+    docker exec minio mc anonymous set public local-docker/tiles
+
+# -- mvtbench seeding
+
+# mvtbench seed files
+seed-bench-files: docker-is-ready
+    rm -rf /tmp/mvtbench
+    cargo build --release
+    ../target/release/bbox-tile-server --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=warn seed --tileset=ne_countries --maxzoom=6
+
+# mvtbench seed MBTiles
+seed-bench-mbtiles: docker-is-ready
+    rm -f /tmp/mvtbench.mbtiles
+    cargo build --release
+    ../target/release/bbox-tile-server --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=warn seed --mb-path=/tmp/mvtbench.mbtiles --tileset=ne_countries --maxzoom=6
+
+# mvtbench seed PMTiles
+seed-bench-pmtiles: docker-is-ready
+    rm -f /tmp/mvtbench.pmtiles
+    cargo build --release
+    ../target/release/bbox-tile-server --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=warn seed --pm-path=/tmp/mvtbench.pmtiles --tileset=ne_countries --maxzoom=6
+
+# -- tile store write benchmarks
+
+seed-write-bench-files:
+    rm -rf /tmp/writebench
+    cargo build --release
+    ../target/release/bbox-tile-server --config=/dev/null --loglevel=warn seed --tile-path=/tmp/writebench --tileset=mvtbench --maxzoom=6 /tmp/mvtbench.pmtiles
+
+seed-write-bench-mbtiles:
+    rm -f /tmp/writebench.mbtiles
+    cargo build --release
+    ../target/release/bbox-tile-server --config=/dev/null --loglevel=info seed --mb-path=/tmp/writebench.mbtiles --tileset=mvtbench --maxzoom=6 /tmp/mvtbench.pmtiles
+
+seed-write-bench-pmtiles:
+    rm -f /tmp/writebench.pmtiles
+    cargo build --release
+    ../target/release/bbox-tile-server --config=/dev/null --loglevel=warn seed --pm-path=/tmp/writebench.pmtiles --tileset=mvtbench --maxzoom=6 /tmp/mvtbench.pmtiles
+
+# -- tile datasource read benchmarks
+
+seed-read-bench-pg: docker-is-ready
+    cargo build --release
+    ../target/release/bbox-tile-server --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=warn seed --no-store --tileset=ne_countries --maxzoom=6
+
+seed-read-bench-mbtiles:
+    cargo build --release
+    ../target/release/bbox-tile-server --config=/dev/null --loglevel=info seed --no-store --tileset=mvtbench --maxzoom=6 /tmp/mvtbench.mbtiles
+
+seed-read-bench-pmtiles:
+    cargo build --release
+    ../target/release/bbox-tile-server --config=/dev/null --loglevel=warn seed --no-store --tileset=mvtbench --maxzoom=6 /tmp/mvtbench.pmtiles
+
+# Run all seeding benchmarks
+seed-all-benchmarks: seed-bench-files seed-bench-mbtiles seed-bench-pmtiles seed-write-bench-files seed-write-bench-mbtiles seed-write-bench-pmtiles seed-read-bench-pg seed-read-bench-mbtiles seed-read-bench-pmtiles
+
+# -- mvtbench test seeding
+
+# mvtbench test seed files
+seed-files: docker-is-ready
+    rm -rf /tmp/mvtbench
+    cargo run -- --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=info seed --tileset=ne_countries --maxzoom=3
+
+# mvtbench test seed MBTiles
+seed-mbtiles: docker-is-ready
+    rm -f /tmp/tilecache.mbtiles
+    cargo run -- --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=info seed --mb-path=/tmp/tilecache.mbtiles --tileset=ne_countries --maxzoom=3
+
+# mvtbench test seed PMTiles
+seed-pmtiles: docker-is-ready
+    rm -f /tmp/tilecache.pmtiles
+    cargo run -- --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=debug seed --pm-path=/tmp/tilecache.pmtiles --tileset=ne_countries --maxzoom=3
+
+export AWS_ACCESS_KEY_ID := "miniostorage"
+export AWS_SECRET_ACCESS_KEY := "miniostorage"
+export S3_ENDPOINT_URL := "http://localhost:9000"
+
+# mvtbench test seed S3
+seed-s3: docker-is-ready
+    cargo run -- --config={{justfile_directory()}}/bbox-mvtbench.toml --loglevel=info seed --tileset=ne_countries --s3-path=s3://tiles --overwrite=true --maxzoom=3
+
+# -- raster seeding
+
+seed-wms-mbtiles:
+    cargo run -- --loglevel=info seed --tileset=gebco --mb-path=/tmp/gebco.mbtile --maxzoom=3
+
+seed-wms-pmtiles:
+    cargo run -- --loglevel=info seed --tileset=gebco --pm-path=/tmp/gebco.pmtile --maxzoom=3
+
+seed-map-mbtiles:
+    cargo run -- --loglevel=warn seed --tileset=ne_extracts --mb-path=/tmp/ne_extracts.mbtile --maxzoom=3
+
+seed-map-pmtiles:
+    cargo run -- --loglevel=warn seed --tileset=ne_extracts --pm-path=/tmp/ne_extracts.pmtile --maxzoom=3

bbox-tile-server/performance.md

@@ -2,35 +2,19 @@
 
 ## Local S3 test setup
 
-Install [MinIO Client](https://github.com/minio/mc):
-
-    cd /usr/local/bin
-    wget https://dl.min.io/client/mc/release/linux-amd64/mc
-    chmod +x mc
-    mc --help
-
-Setup storage directory:
-
-    mkdir s3data
-
 Run MinIO:
 
-    docker run --security-opt label=disable -d --rm --name minio -p 9000:9000 -p 9001:9001 -v $PWD/s3data:/data -e MINIO_REGION_NAME=my-region -e MINIO_ROOT_USER=miniostorage -e MINIO_ROOT_PASSWORD=miniostorage minio/minio server /data --console-address ":9001"
+    just start-minio
 
 Setup Bucket:
 
-    export AWS_ACCESS_KEY_ID=miniostorage
-    export AWS_SECRET_ACCESS_KEY=miniostorage
-
-    mc config host add local-docker http://localhost:9000 miniostorage miniostorage
-    mc mb local-docker/tiles
-    mc policy set public local-docker/tiles
+    just setup-minio
 
 Access MinIO Console: http://localhost:9001
 
 Stop MinIO:
 
-    docker stop minio
+    just stop-minio
 
 
 ## Seeding tests
@@ -43,6 +27,10 @@ Local file seeding test:
 
     ../target/release/bbox-tile-server seed --tileset=gebco --base-dir=/tmp/tiles --maxzoom=2
 
+or mvtbench:
+
+    just seed-s3
+
 Set S3 env vars:
 
     export S3_ENDPOINT_URL="http://localhost:9000"
@@ -54,6 +42,21 @@ Run:
     ../target/release/bbox-tile-server seed --tileset=ne_extracts --s3-path=s3://tiles --maxzoom=5
 
 
+## MVT benchmark
+
+https://github.com/pka/mvt-benchmark
+
+Start DB:
+
+    just start-db
+
+Tile seeding benchmarks:
+
+    just seed-bench-files
+    just seed-bench-mbtiles
+    just seed-bench-pmtiles
+
+
 ## S3 upload benchmarks
 
 ### s3cmd