Merge pull request #85 from pka/auto-dedup

Automatic tile deduplication

bbox-tile-server/Cargo.toml

@@ -29,6 +29,7 @@ async-trait = { workspace = true }
 bbox-asset-server = { path = "../bbox-asset-server", version = "0.6.1", optional = true }
 bbox-core = { path = "../bbox-core", version = "0.6.1" }
 bbox-map-server = { path = "../bbox-map-server", version = "0.6.1", optional = true }
+blake3 = "1.5.4"
 bytes = "1.1.0"
 chrono = { workspace = true }
 clap = { workspace = true }

bbox-tile-server/src/config.rs

@@ -328,6 +328,18 @@ pub enum TileStoreCfg {
 pub struct FileStoreCfg {
     /// Base directory, tileset name will be appended
     pub base_dir: PathBuf,
+    /// Tile deduplication method.
+    ///
+    /// Defaults to `Hardlink` for seeding and `None` for serving.
+    pub deduplication: Option<FileDedupCfg>,
+}
+
+/// Tile deduplication method.
+#[derive(Deserialize, Serialize, Clone, Debug)]
+pub enum FileDedupCfg {
+    Off,
+    Hardlink,
+    // Softlink,
 }
 
 impl FileStoreCfg {
@@ -377,6 +389,7 @@ impl TileStoreCfg {
         if let Some(path) = &args.tile_path {
             let cache_cfg = TileStoreCfg::Files(FileStoreCfg {
                 base_dir: path.into(),
+                deduplication: None,
             });
             Some(cache_cfg)
         } else if let Some(s3_path) = &args.s3_path {
@@ -503,6 +516,7 @@ impl From<t_rex::ApplicationCfg> for TileServiceCfg {
                 if let Some(fcache) = cache.file {
                     Some(TileStoreCfg::Files(FileStoreCfg {
                         base_dir: fcache.base.into(),
+                        deduplication: None,
                     }))
                 } else {
                     None

bbox-tile-server/src/mbtiles_ds.rs

@@ -97,9 +97,7 @@ impl MbtilesDatasource {
             // PRAGMA page_size = 512
             // PRAGMA encoding = 'UTF-8'
             // VACUUM
-            init_mbtiles_schema(&mut conn, MbtType::Flat).await?;
-            // MbtType::Normalized { hash_view: true } does not work because extension functions are
-            // registered only when using Mbtiles::open but not via SqlitePool.
+            init_mbtiles_schema(&mut conn, MbtType::Normalized { hash_view: true }).await?;
             // metadata content example:
             // ('name','Tilemaker to OpenTileMaps schema');
             // ('type','baselayer');

bbox-tile-server/src/seed.rs

@@ -89,6 +89,17 @@ impl TileService {
             None // tms.xy_bbox()
         };
 
+        // Number of worker threads (size >= #cores).
+        let threads = args.threads.unwrap_or(num_cpus::get());
+
+        let minzoom = args.minzoom.unwrap_or(0);
+        let maxzoom = args.maxzoom.unwrap_or(tms.maxzoom());
+        let griditer: Box<dyn TileIterator + Send> = if let Some(bbox) = bbox {
+            Box::new(tms.xyz_iterator(&bbox, minzoom, maxzoom))
+        } else {
+            Box::new(tms.hilbert_iterator(minzoom, maxzoom))
+        };
+
         let ts = tileset.clone();
         let Some(cache_cfg) = &ts.cache_config() else {
             return Err(
@@ -102,18 +113,9 @@ impl TileService {
             .into());
         };
         let compression = tile_store.compression();
-        let tile_writer = Arc::new(tile_store.setup_writer().await?);
+        // let n_tiles = ((1 << maxzoom) as usize).pow(2);
+        let tile_writer = Arc::new(tile_store.setup_writer(true).await?);
 
-        // Number of worker threads (size >= #cores).
-        let threads = args.threads.unwrap_or(num_cpus::get());
-
-        let minzoom = args.minzoom.unwrap_or(0);
-        let maxzoom = args.maxzoom.unwrap_or(tms.maxzoom());
-        let griditer: Box<dyn TileIterator + Send> = if let Some(bbox) = bbox {
-            Box::new(tms.xyz_iterator(&bbox, minzoom, maxzoom))
-        } else {
-            Box::new(tms.hilbert_iterator(minzoom, maxzoom))
-        };
         info!("Seeding tiles from level {minzoom} to {maxzoom}");
 
         // We setup different pipelines for certain scenarios.

bbox-tile-server/src/service.rs

@@ -326,8 +326,8 @@ impl TileService {
 impl TileSet {
     pub async fn setup_tile_store(&mut self) -> Result<(), TileStoreError> {
         if let Some(ts) = &self.tile_store {
-            self.cache_reader = Some(ts.setup_reader().await?);
-            self.cache_writer = Some(ts.setup_writer().await?);
+            self.cache_reader = Some(ts.setup_reader(false).await?);
+            self.cache_writer = Some(ts.setup_writer(false).await?);
         }
         Ok(())
     }