From df59dcd93b63bbdbc430e50310d82a624cb9e0c4 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 15 Dec 2023 23:56:54 -0500 Subject: [PATCH 1/2] Split Clap code from `MbtilesCopier` into `CopyArgs` This allows easier reuse of the copy functionality without affecting CLI commands. --- mbtiles/src/bin/mbtiles.rs | 153 ++++++++++++++++++++++--------------- mbtiles/src/copier.rs | 24 +----- mbtiles/src/mbtiles.rs | 4 +- mbtiles/src/validation.rs | 6 +- 4 files changed, 94 insertions(+), 93 deletions(-) diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 76c11048a..ebfb5e631 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -2,7 +2,11 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use log::error; -use mbtiles::{apply_patch, AggHashType, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier}; +use mbtiles::{ + apply_patch, AggHashType, CopyDuplicateMode, IntegrityCheckType, MbtResult, MbtTypeCli, + Mbtiles, MbtilesCopier, +}; +use tilejson::Bounds; #[derive(Parser, PartialEq, Debug)] #[command( @@ -31,7 +35,7 @@ enum Commands { file: PathBuf, }, /// Gets a single value from the MBTiles metadata table. - #[command(name = "meta-get")] + #[command(name = "meta-get", alias = "get-meta")] MetaGetValue { /// MBTiles file to read a value from file: PathBuf, @@ -39,7 +43,7 @@ enum Commands { key: String, }, /// Sets a single value in the MBTiles metadata table or deletes it if no value. - #[command(name = "meta-set")] + #[command(name = "meta-set", alias = "set-meta")] MetaSetValue { /// MBTiles file to modify file: PathBuf, @@ -50,7 +54,7 @@ enum Commands { }, /// Copy tiles from one mbtiles file to another. #[command(name = "copy", alias = "cp")] - Copy(MbtilesCopier), + Copy(CopyArgs), /// Apply diff file generated from 'copy' command #[command(name = "apply-patch", alias = "apply-diff")] ApplyPatch { @@ -76,6 +80,43 @@ enum Commands { }, } +#[derive(Clone, Default, PartialEq, Debug, clap::Args)] +pub struct CopyArgs { + /// MBTiles file to read from + pub src_file: PathBuf, + /// MBTiles file to write to + pub dst_file: PathBuf, + /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source + #[arg(long, alias = "dst-type", alias = "dst_type", value_name = "SCHEMA")] + pub mbtiles_type: Option, + /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists + #[arg(long, value_enum)] + pub on_duplicate: Option, + /// Minimum zoom level to copy + #[arg(long, conflicts_with("zoom_levels"))] + pub min_zoom: Option, + /// Maximum zoom level to copy + #[arg(long, conflicts_with("zoom_levels"))] + pub max_zoom: Option, + /// List of zoom levels to copy + #[arg(long, value_delimiter = ',')] + pub zoom_levels: Vec, + /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. + #[arg(long)] + pub bbox: Vec, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. + #[arg(long, conflicts_with("apply_patch"))] + pub diff_with_file: Option, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. + #[arg(long, conflicts_with("diff_with_file"))] + pub apply_patch: Option, + /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. + #[arg(long)] + pub skip_agg_tiles_hash: bool, +} + #[tokio::main] async fn main() { let env = env_logger::Env::default().default_filter_or("mbtiles=info"); @@ -105,6 +146,20 @@ async fn main_int() -> anyhow::Result<()> { meta_set_value(file.as_path(), &key, value.as_deref()).await?; } Commands::Copy(opts) => { + let opts = MbtilesCopier { + src_file: opts.src_file, + dst_file: opts.dst_file, + dst_type_cli: opts.mbtiles_type, + dst_type: None, + on_duplicate: opts.on_duplicate, + min_zoom: opts.min_zoom, + max_zoom: opts.max_zoom, + zoom_levels: opts.zoom_levels, + bbox: opts.bbox, + diff_with_file: opts.diff_with_file, + apply_patch: opts.apply_patch, + skip_agg_tiles_hash: opts.skip_agg_tiles_hash, + }; opts.run().await?; } Commands::ApplyPatch { @@ -176,7 +231,7 @@ mod tests { use clap::error::ErrorKind; use clap::Parser; - use mbtiles::{CopyDuplicateMode, MbtilesCopier}; + use mbtiles::CopyDuplicateMode; use super::*; use crate::Commands::{ApplyPatch, Copy, MetaGetValue, MetaSetValue, Validate}; @@ -198,20 +253,17 @@ mod tests { Args::parse_from(["mbtiles", "copy", "src_file", "dst_file"]), Args { verbose: false, - command: Copy(MbtilesCopier::new( - PathBuf::from("src_file"), - PathBuf::from("dst_file") - )) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + ..Default::default() + }) } ); } #[test] fn test_copy_min_max_zoom_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.min_zoom = Some(1); - opt.max_zoom = Some(100); - let args = Args::parse_from([ "mbtiles", "copy", @@ -226,7 +278,13 @@ mod tests { args, Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + min_zoom: Some(1), + max_zoom: Some(100), + ..Default::default() + }) } ); } @@ -271,8 +329,6 @@ mod tests { #[test] fn test_copy_zoom_levels_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.zoom_levels.extend(&[3, 7, 1]); assert_eq!( Args::parse_from([ "mbtiles", @@ -284,15 +340,18 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + zoom_levels: vec![3, 7, 1], + ..Default::default() + }) } ); } #[test] fn test_copy_diff_with_file_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.diff_with_file = Some(PathBuf::from("no_file")); assert_eq!( Args::parse_from([ "mbtiles", @@ -304,15 +363,18 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + diff_with_file: Some(PathBuf::from("no_file")), + ..Default::default() + }) } ); } #[test] fn test_copy_diff_with_override_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Override); assert_eq!( Args::parse_from([ "mbtiles", @@ -324,47 +386,12 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) - } - ); - } - - #[test] - fn test_copy_diff_with_ignore_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Ignore); - assert_eq!( - Args::parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--on-duplicate", - "ignore" - ]), - Args { - verbose: false, - command: Copy(opt) - } - ); - } - - #[test] - fn test_copy_diff_with_abort_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Abort); - assert_eq!( - Args::parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--on-duplicate", - "abort" - ]), - Args { - verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + on_duplicate: Some(CopyDuplicateMode::Override), + ..Default::default() + }) } ); } diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index a5a576f98..e59763ab5 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -1,8 +1,6 @@ use std::fmt::Write as _; use std::path::PathBuf; -#[cfg(feature = "cli")] -use clap::{Args, ValueEnum}; use enum_display::EnumDisplay; use itertools::Itertools as _; use log::{debug, info, trace}; @@ -24,7 +22,7 @@ use crate::{ #[derive(PartialEq, Eq, Debug, Clone, Copy, EnumDisplay, Serialize, Deserialize)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum CopyDuplicateMode { Override, Ignore, @@ -43,52 +41,32 @@ impl CopyDuplicateMode { } #[derive(Clone, Default, PartialEq, Debug)] -#[cfg_attr(feature = "cli", derive(Args))] pub struct MbtilesCopier { /// MBTiles file to read from pub src_file: PathBuf, /// MBTiles file to write to pub dst_file: PathBuf, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source - #[cfg_attr( - feature = "cli", - arg( - long = "mbtiles-type", - alias = "dst-type", - alias = "dst_type", - value_name = "SCHEMA", - value_enum - ) - )] pub dst_type_cli: Option, /// Destination type with options - #[cfg_attr(feature = "cli", arg(skip))] pub dst_type: Option, /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists - #[cfg_attr(feature = "cli", arg(long, value_enum))] pub on_duplicate: Option, /// Minimum zoom level to copy - #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] pub min_zoom: Option, /// Maximum zoom level to copy - #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] pub max_zoom: Option, /// List of zoom levels to copy - #[cfg_attr(feature = "cli", arg(long, value_delimiter = ','))] pub zoom_levels: Vec, /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. - #[cfg_attr(feature = "cli", arg(long))] pub bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. - #[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))] pub diff_with_file: Option, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. - #[cfg_attr(feature = "cli", arg(long, conflicts_with("diff_with_file")))] pub apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. - #[cfg_attr(feature = "cli", arg(long))] pub skip_agg_tiles_hash: bool, } diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index de9bfe030..64dcb16e5 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -2,8 +2,6 @@ use std::ffi::OsStr; use std::fmt::{Display, Formatter}; use std::path::Path; -#[cfg(feature = "cli")] -use clap::ValueEnum; use enum_display::EnumDisplay; use log::debug; use serde::{Deserialize, Serialize}; @@ -16,7 +14,7 @@ use crate::{invert_y_value, CopyDuplicateMode, MbtType}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum MbtTypeCli { Flat, FlatWithHash, diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index d222c7df9..57e9ebbc0 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -1,8 +1,6 @@ use std::collections::HashSet; use std::str::from_utf8; -#[cfg(feature = "cli")] -use clap::ValueEnum; use enum_display::EnumDisplay; use log::{debug, info, warn}; use martin_tile_utils::{Format, TileInfo, MAX_ZOOM}; @@ -52,7 +50,7 @@ impl MbtType { #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum IntegrityCheckType { #[default] Quick, @@ -62,7 +60,7 @@ pub enum IntegrityCheckType { #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum AggHashType { /// Verify that the aggregate tiles hash value in the metadata table matches the computed value. Used by default. #[default] From 52adb6cad0749ceada10269da8ecdb34e8387fa2 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 16 Dec 2023 00:06:05 -0500 Subject: [PATCH 2/2] ignore maplibre/martin link check --- .github/files/markdown.links.config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index fad1deb32..eaff6f07c 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -15,6 +15,9 @@ { "pattern": "^http://localhost" }, + { + "pattern": "^https://ghcr.io/maplibre/martin($|/|\\?)" + }, { "pattern": "^http://opensource.org" }