From 9702e7f910c2fe8f39c2ef7d90804a08f4029d76 Mon Sep 17 00:00:00 2001 From: Sergei Gureev Date: Tue, 18 Oct 2022 15:00:28 +0300 Subject: [PATCH] refactor: Move Map and related types to provider pkg This is needed to access configured map parameters from the provider's initialization code to inspect geom type. Unfortunately, config pkg depends on the provider pkg, so simply referencing these types creates a dependency loop. --- atlas/map.go | 11 +- cmd/internal/register/maps.go | 6 +- cmd/internal/register/maps_test.go | 24 ++-- cmd/internal/register/providers.go | 4 +- cmd/internal/register/providers_test.go | 2 +- cmd/tegola/cmd/root.go | 2 +- cmd/tegola_lambda/main.go | 2 +- config/config.go | 131 +++-------------- config/config_test.go | 143 ++++++++++--------- config/errors.go | 48 +++---- provider/debug/debug.go | 12 +- provider/gpkg/cgo_test.go | 3 +- provider/gpkg/gpkg.go | 4 +- provider/gpkg/gpkg_register.go | 5 +- provider/gpkg/gpkg_register_internal_test.go | 3 +- provider/gpkg/gpkg_test.go | 12 +- provider/map.go | 14 ++ provider/map_layer.go | 51 +++++++ provider/mvt_provider.go | 4 +- provider/paramater_decoders.go | 19 +++ provider/postgis/postgis.go | 79 +++++----- provider/postgis/postgis_internal_test.go | 16 ++- provider/postgis/postgis_test.go | 6 +- provider/postgis/register.go | 8 +- provider/postgis/util.go | 23 ++- provider/postgis/util_internal_test.go | 122 ---------------- provider/provider.go | 57 +------- provider/provider_test.go | 4 +- provider/query_parameter.go | 65 +++++++++ provider/query_parameter_value.go | 76 ++++++++++ provider/query_parameter_value_test.go | 137 ++++++++++++++++++ provider/test/emptycollection/provider.go | 4 +- provider/test/provider.go | 8 +- server/handle_map_layer_zxy.go | 47 +++--- tile.go | 4 +- 35 files changed, 636 insertions(+), 520 deletions(-) create mode 100644 provider/map.go create mode 100644 provider/map_layer.go create mode 100644 provider/paramater_decoders.go create mode 100644 provider/query_parameter.go create mode 100644 provider/query_parameter_value.go create mode 100644 provider/query_parameter_value_test.go diff --git a/atlas/map.go b/atlas/map.go index 5f1385c8b..aab8e79de 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -9,7 +9,6 @@ import ( "strings" "sync" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/golang/protobuf/proto" @@ -56,7 +55,7 @@ type Map struct { Center [3]float64 Layers []Layer // Params holds configured query parameters - Params []config.QueryParameter + Params []provider.QueryParameter SRID uint64 // MVT output values @@ -120,7 +119,7 @@ func (m Map) AddDebugLayers() Map { m.Layers = layers // setup a debug provider - debugProvider, _ := debug.NewTileProvider(dict.Dict{}) + debugProvider, _ := debug.NewTileProvider(dict.Dict{}, nil) m.Layers = append(layers, []Layer{ { @@ -183,7 +182,7 @@ func (m Map) FilterLayersByName(names ...string) Map { return m } -func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // get the list of our layers ptile := provider.NewTile(tile.Z, tile.X, tile.Y, uint(m.TileBuffer), uint(m.SRID)) @@ -200,7 +199,7 @@ func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, param // encodeMVTTile will encode the given tile into mvt format // TODO (arolek): support for max zoom -func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // tile container var mvtTile mvt.Tile @@ -380,7 +379,7 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[st } // Encode will encode the given tile into mvt format -func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { var ( tileBytes []byte err error diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index 69dec943e..bb4e62c9f 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -11,7 +11,7 @@ import ( "github.com/go-spatial/tegola/provider" ) -func webMercatorMapFromConfigMap(cfg config.Map) (newMap atlas.Map) { +func webMercatorMapFromConfigMap(cfg provider.Map) (newMap atlas.Map) { newMap = atlas.NewWebMercatorMap(string(cfg.Name)) newMap.Attribution = SanitizeAttribution(string(cfg.Attribution)) newMap.Params = cfg.Parameters @@ -47,7 +47,7 @@ func layerInfosFindByName(infos []provider.LayerInfo, name string) provider.Laye return nil } -func atlasLayerFromConfigLayer(cfg *config.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { +func atlasLayerFromConfigLayer(cfg *provider.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { var ( // providerLayer is primary used for error reporting. providerLayer = string(cfg.ProviderLayer) @@ -124,7 +124,7 @@ func selectProvider(name string, mapName string, newMap *atlas.Map, providers ma } // Maps registers maps with with atlas -func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.TilerUnion) error { +func Maps(a *atlas.Atlas, maps []provider.Map, providers map[string]provider.TilerUnion) error { var ( layerer provider.Layerer diff --git a/cmd/internal/register/maps_test.go b/cmd/internal/register/maps_test.go index abe05dcc1..5573bb8c4 100644 --- a/cmd/internal/register/maps_test.go +++ b/cmd/internal/register/maps_test.go @@ -6,15 +6,15 @@ import ( "github.com/go-spatial/tegola/atlas" "github.com/go-spatial/tegola/cmd/internal/register" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" ) func TestMaps(t *testing.T) { type tcase struct { atlas atlas.Atlas - maps []config.Map + maps []provider.Map providers []dict.Dict expectedErr error } @@ -29,7 +29,7 @@ func TestMaps(t *testing.T) { provArr[i] = tc.providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, tc.maps) if err != nil { t.Errorf("unexpected err: %v", err) return @@ -45,10 +45,10 @@ func TestMaps(t *testing.T) { tests := map[string]tcase{ "provider layer invalid": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar", }, @@ -67,10 +67,10 @@ func TestMaps(t *testing.T) { }, }, "provider not found": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar.baz", }, @@ -82,10 +82,10 @@ func TestMaps(t *testing.T) { }, }, "provider layer not registered with provider": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.bar", }, @@ -105,10 +105,10 @@ func TestMaps(t *testing.T) { }, }, "default tags": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.debug-tile-outline", DefaultTags: env.Dict{ @@ -126,7 +126,7 @@ func TestMaps(t *testing.T) { }, }, "success": { - maps: []config.Map{}, + maps: []provider.Map{}, providers: []dict.Dict{ { "name": "test", diff --git a/cmd/internal/register/providers.go b/cmd/internal/register/providers.go index 1eb16faa9..4553b9fc1 100644 --- a/cmd/internal/register/providers.go +++ b/cmd/internal/register/providers.go @@ -33,7 +33,7 @@ func (e ErrProviderTypeInvalid) Error() string { } // Providers registers data provider backends -func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) { +func Providers(providers []dict.Dicter, maps []provider.Map) (map[string]provider.TilerUnion, error) { // holder for registered providers registeredProviders := map[string]provider.TilerUnion{} @@ -72,7 +72,7 @@ func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) } // register the provider - prov, err := provider.For(ptype, p) + prov, err := provider.For(ptype, p, maps) if err != nil { return registeredProviders, err } diff --git a/cmd/internal/register/providers_test.go b/cmd/internal/register/providers_test.go index c1f62a45b..9f8d44410 100644 --- a/cmd/internal/register/providers_test.go +++ b/cmd/internal/register/providers_test.go @@ -23,7 +23,7 @@ func TestProviders(t *testing.T) { provArr[i] = tc.config[i] } - _, err = register.Providers(provArr) + _, err = register.Providers(provArr, nil) if tc.expectedErr != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("invalid error. expected: %v, got %v", tc.expectedErr, err.Error()) diff --git a/cmd/tegola/cmd/root.go b/cmd/tegola/cmd/root.go index 3e41e8c34..177638ae2 100644 --- a/cmd/tegola/cmd/root.go +++ b/cmd/tegola/cmd/root.go @@ -121,7 +121,7 @@ func initConfig(configFile string, cacheRequired bool, logLevel string, logger s provArr[i] = conf.Providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, conf.Maps) if err != nil { return fmt.Errorf("could not register providers: %v", err) } diff --git a/cmd/tegola_lambda/main.go b/cmd/tegola_lambda/main.go index 4d1e19f4a..cf96b11ab 100644 --- a/cmd/tegola_lambda/main.go +++ b/cmd/tegola_lambda/main.go @@ -66,7 +66,7 @@ func init() { } // register the providers - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, nil) if err != nil { log.Fatal(err) } diff --git a/config/config.go b/config/config.go index 09685c755..e4dd0ca48 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "strconv" "strings" "time" @@ -46,22 +45,6 @@ var ReservedTokens = map[string]struct{}{ GeomTypeToken: {}, } -// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters -var ParamTypeDecoders = map[string]func(string) (interface{}, error){ - "int": func(s string) (interface{}, error) { - return strconv.Atoi(s) - }, - "float": func(s string) (interface{}, error) { - return strconv.ParseFloat(s, 32) - }, - "string": func(s string) (interface{}, error) { - return s, nil - }, - "bool": func(s string) (interface{}, error) { - return strconv.ParseBool(s) - }, -} - var blacklistHeaders = []string{"content-encoding", "content-length", "content-type"} // Config represents a tegola config file. @@ -82,8 +65,8 @@ type Config struct { // 2. type -- this is the name the provider modules register // themselves under. (e.g. postgis, gpkg, mvt_postgis ) // Note: Use the type to figure out if the provider is a mvt or std provider - Providers []env.Dict `toml:"providers"` - Maps []Map `toml:"maps"` + Providers []env.Dict `toml:"providers"` + Maps []provider.Map `toml:"maps"` } // Webserver represents the config options for the webserver part of Tegola @@ -96,47 +79,36 @@ type Webserver struct { SSLKey env.String `toml:"ssl_key"` } -// A Map represents a map in the Tegola Config file. -type Map struct { - Name env.String `toml:"name"` - Attribution env.String `toml:"attribution"` - Bounds []env.Float `toml:"bounds"` - Center [3]env.Float `toml:"center"` - Layers []MapLayer `toml:"layers"` - Parameters []QueryParameter `toml:"params"` - TileBuffer *env.Int `toml:"tile_buffer"` -} - -// ValidateParams ensures configured params don't conflict with existing +// ValidateAndRegisterParams ensures configured params don't conflict with existing // query tokens or have overlapping names -func (m Map) ValidateParams() error { - if len(m.Parameters) == 0 { +func ValidateAndRegisterParams(mapName string, params []provider.QueryParameter) error { + if len(params) == 0 { return nil } usedNames := make(map[string]struct{}) usedTokens := make(map[string]struct{}) - for _, param := range m.Parameters { - if _, ok := ParamTypeDecoders[param.Type]; !ok { + for _, param := range params { + if _, ok := provider.ParamTypeDecoders[param.Type]; !ok { return ErrParamUnknownType{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultSQL) > 0 && len(param.DefaultValue) > 0 { return ErrParamTwoDefaults{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultValue) > 0 { - decoderFn := ParamTypeDecoders[param.Type] + decoderFn := provider.ParamTypeDecoders[param.Type] if _, err := decoderFn(param.DefaultValue); err != nil { return ErrParamInvalidDefault{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } @@ -144,28 +116,28 @@ func (m Map) ValidateParams() error { if _, ok := ReservedTokens[param.Token]; ok { return ErrParamTokenReserved{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if !provider.ParameterTokenRegexp.MatchString(param.Token) { return ErrParamBadTokenName{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if _, ok := usedNames[param.Name]; ok { - return ErrParamNameDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateName{ + MapName: string(mapName), Parameter: param, } } if _, ok := usedTokens[param.Token]; ok { - return ErrParamTokenDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateToken{ + MapName: string(mapName), Parameter: param, } } @@ -182,69 +154,6 @@ func (m Map) ValidateParams() error { return nil } -// MapLayer represents a the config for a layer in a map -type MapLayer struct { - // Name is optional. If it's not defined the name of the ProviderLayer will be used. - // Name can also be used to group multiple ProviderLayers under the same namespace. - Name env.String `toml:"name"` - ProviderLayer env.String `toml:"provider_layer"` - MinZoom *env.Uint `toml:"min_zoom"` - MaxZoom *env.Uint `toml:"max_zoom"` - DefaultTags env.Dict `toml:"default_tags"` - // DontSimplify indicates whether feature simplification should be applied. - // We use a negative in the name so the default is to simplify - DontSimplify env.Bool `toml:"dont_simplify"` - // DontClip indicates whether feature clipping should be applied. - // We use a negative in the name so the default is to clipping - DontClip env.Bool `toml:"dont_clip"` - // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. - // We use a negative in the name so the default is to clean - DontClean env.Bool `toml:"dont_clean"` -} - -// ProviderLayerName returns the names of the layer and provider or an error -func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { - // split the provider layer (syntax is provider.layer) - plParts := strings.Split(string(ml.ProviderLayer), ".") - if len(plParts) != 2 { - return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} - } - return plParts[0], plParts[1], nil -} - -// GetName will return the user-defined Layer name from the config, -// or if the name is empty, return the name of the layer associated with -// the provider -func (ml MapLayer) GetName() (string, error) { - if ml.Name != "" { - return string(ml.Name), nil - } - _, name, err := ml.ProviderLayerName() - return name, err -} - -// QueryParameter represents an HTTP query parameter specified for use with -// a given map instance. -type QueryParameter struct { - Name string `toml:"name"` - Token string `toml:"token"` - Type string `toml:"type"` - SQL string `toml:"sql"` - // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or - // default_value can be specified - DefaultSQL string `toml:"default_sql"` - DefaultValue string `toml:"default_value"` -} - -// Normalize normalizes param and sets default values -func (param *QueryParameter) Normalize() { - param.Token = strings.ToUpper(param.Token) - - if len(param.SQL) == 0 { - param.SQL = "?" - } -} - // Validate checks the config for issues func (c *Config) Validate() error { @@ -286,16 +195,16 @@ func (c *Config) Validate() error { } // check for map layer name / zoom collisions // map of layers to providers - mapLayers := map[string]map[string]MapLayer{} + mapLayers := map[string]map[string]provider.MapLayer{} for mapKey, m := range c.Maps { // validate any declared query parameters - if err := m.ValidateParams(); err != nil { + if err := ValidateAndRegisterParams(string(m.Name), m.Parameters); err != nil { return err } if _, ok := mapLayers[string(m.Name)]; !ok { - mapLayers[string(m.Name)] = map[string]MapLayer{} + mapLayers[string(m.Name)] = map[string]provider.MapLayer{} } // Set current provider to empty, for MVT providers diff --git a/config/config_test.go b/config/config_test.go index 8ae0f59e9..5bf34ced6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" _ "github.com/go-spatial/tegola/provider/debug" _ "github.com/go-spatial/tegola/provider/postgis" _ "github.com/go-spatial/tegola/provider/test" @@ -167,14 +168,14 @@ func TestParse(t *testing.T) { name = "param2" token = "!PARAM2!" type = "int" - sql = "AND ANSWER = ?" + sql = "AND answer = ?" default_value = "42" [[maps.params]] name = "param3" token = "!PARAM3!" type = "float" - default_sql = "AND PI = 3.1415926" + default_sql = "AND pi = 3.1415926" `, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(12)), @@ -210,14 +211,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(12)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -227,7 +228,7 @@ func TestParse(t *testing.T) { DontClean: true, }, }, - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param1", Token: "!PARAM1!", @@ -238,7 +239,7 @@ func TestParse(t *testing.T) { Name: "param2", Token: "!PARAM2!", Type: "int", - SQL: "AND ANSWER = ?", + SQL: "AND answer = ?", DefaultValue: "42", }, { @@ -246,7 +247,7 @@ func TestParse(t *testing.T) { Token: "!PARAM3!", Type: "float", SQL: "?", - DefaultSQL: "AND PI = 3.1415926", + DefaultSQL: "AND pi = 3.1415926", }, }, }, @@ -360,14 +361,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{ENV_TEST_CENTER_X, ENV_TEST_CENTER_Y, ENV_TEST_CENTER_Z}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: ENV_TEST_PROVIDER_LAYER, @@ -388,7 +389,7 @@ func TestParse(t *testing.T) { Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -542,13 +543,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: nil, @@ -586,13 +587,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(0), @@ -672,13 +673,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -736,13 +737,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -806,13 +807,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -830,7 +831,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -889,13 +890,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", }, @@ -906,7 +907,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider2.water", }, @@ -958,13 +959,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1076,11 +1077,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1097,11 +1098,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1125,11 +1126,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1153,11 +1154,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bad.water_default_z", }, @@ -1182,11 +1183,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1214,11 +1215,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "stdprovider1.water_default_z", }, @@ -1230,12 +1231,12 @@ func TestValidate(t *testing.T) { }, }, }, - "13 reserved parameter token": { + "13 reserved token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "bad_param", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1247,7 +1248,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTokenReserved{ MapName: "bad_param", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1256,10 +1257,10 @@ func TestValidate(t *testing.T) { }, "13 duplicate parameter name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_name", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1274,21 +1275,21 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamNameDuplicate{ + expectedErr: config.ErrParamDuplicateName{ MapName: "dupe_param_name", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!PARAM2!", Type: "int", }, }, }, - "13 duplicate parameter token": { + "13 duplicate token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1303,9 +1304,9 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamTokenDuplicate{ + expectedErr: config.ErrParamDuplicateToken{ MapName: "dupe_param_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param2", Token: "!PARAM!", Type: "int", @@ -1314,10 +1315,10 @@ func TestValidate(t *testing.T) { }, "13 parameter unknown type": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_param_type", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1329,7 +1330,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamUnknownType{ MapName: "unknown_param_type", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "foo", @@ -1338,10 +1339,10 @@ func TestValidate(t *testing.T) { }, "13 parameter two defaults": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_two_defaults", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1355,7 +1356,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTwoDefaults{ MapName: "unknown_two_defaults", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "string", @@ -1366,11 +1367,11 @@ func TestValidate(t *testing.T) { }, "13 parameter invalid default": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_default", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1383,7 +1384,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamInvalidDefault{ MapName: "parameter_invalid_default", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1393,10 +1394,10 @@ func TestValidate(t *testing.T) { }, "13 invalid token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!Token with spaces!", @@ -1408,7 +1409,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamBadTokenName{ MapName: "parameter_invalid_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!Token with spaces!", Type: "int", @@ -1444,14 +1445,14 @@ func TestConfigureTileBuffers(t *testing.T) { tests := map[string]tcase{ "1 tilebuffer is not set": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(64)), @@ -1462,7 +1463,7 @@ func TestConfigureTileBuffers(t *testing.T) { "2 tilebuffer is set in global section": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, @@ -1473,7 +1474,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(32)), @@ -1487,7 +1488,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, "3 tilebuffer is set in map section": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1499,7 +1500,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1514,7 +1515,7 @@ func TestConfigureTileBuffers(t *testing.T) { "4 tilebuffer is set in global and map sections": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1523,7 +1524,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), diff --git a/config/errors.go b/config/errors.go index be6e5a59f..ded68ddfb 100644 --- a/config/errors.go +++ b/config/errors.go @@ -3,6 +3,8 @@ package config import ( "fmt" "strings" + + "github.com/go-spatial/tegola/provider" ) type ErrMapNotFound struct { @@ -15,78 +17,76 @@ func (e ErrMapNotFound) Error() string { type ErrParamTokenReserved struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTokenReserved) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing reserved token %s", + return fmt.Sprintf("config: map %s parameter %s uses a reserved token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } -type ErrParamNameDuplicate struct { +type ErrParamDuplicateName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamNameDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares duplicate parameter with name %s", +func (e ErrParamDuplicateName) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a name already used by another parameter", e.MapName, e.Parameter.Name) } -type ErrParamTokenDuplicate struct { +type ErrParamDuplicateToken struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamTokenDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares existing parameter token %s in param %s", +func (e ErrParamDuplicateToken) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a token name %s already used by another parameter", e.MapName, e.Parameter.Token, e.Parameter.Name) } type ErrParamUnknownType struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamUnknownType) Error() string { - validTypes := make([]string, len(ParamTypeDecoders)) - i := 0 - for k := range ParamTypeDecoders { - validTypes[i] = k - i++ + validTypes := make([]string, 0, len(provider.ParamTypeDecoders)) + for k := range provider.ParamTypeDecoders { + validTypes = append(validTypes, k) } - return fmt.Sprintf("config: map %s has type %s in param %s, which is not one of the known types: %s", - e.MapName, e.Parameter.Type, e.Parameter.Name, strings.Join(validTypes, ",")) + return fmt.Sprintf("config: map %s parameter %s has an unknown type %s, must be one of: %s", + e.MapName, e.Parameter.Name, e.Parameter.Type, strings.Join(validTypes, ",")) } type ErrParamTwoDefaults struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTwoDefaults) Error() string { - return fmt.Sprintf("config: map %s has both default_value and default_sql defined in param %s", + return fmt.Sprintf("config: map %s parameter %s has both default_value and default_sql", e.MapName, e.Parameter.Name) } type ErrParamInvalidDefault struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamInvalidDefault) Error() string { - return fmt.Sprintf("config: map %s has default value in param %s that doesn't match the parameter's type %s", + return fmt.Sprintf("config: map %s parameter %s has a default value that is invalid for type %s", e.MapName, e.Parameter.Name, e.Parameter.Type) } type ErrParamBadTokenName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamBadTokenName) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing token with an invalid name %s", + return fmt.Sprintf("config: map %s parameter %s has an invalid token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } diff --git a/provider/debug/debug.go b/provider/debug/debug.go index 212eab06e..ba96f41f1 100644 --- a/provider/debug/debug.go +++ b/provider/debug/debug.go @@ -26,25 +26,21 @@ func init() { } // NewProvider Setups a debug provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { return &Provider{}, nil } // Provider provides the debug provider type Provider struct{} -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := tile.Extent() - params := make([]string, len(queryParams)) - i := 0 + params := make([]string, 0, len(queryParams)) for _, param := range queryParams { - for k, v := range param.RawValues { - params[i] = fmt.Sprintf("%s=%s", k, v) - i++ - } + params = append(params, fmt.Sprintf("%s=%s", param.RawParam, param.RawValue)) } paramsStr := strings.Join(params, " ") diff --git a/provider/gpkg/cgo_test.go b/provider/gpkg/cgo_test.go index 8a64fb9b1..9d7f7902e 100644 --- a/provider/gpkg/cgo_test.go +++ b/provider/gpkg/cgo_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -11,7 +12,7 @@ import ( // This is a test to just see that the init function is doing something. func TestNewProviderStartup(t *testing.T) { - _, err := NewTileProvider(dict.Dict{}) + _, err := NewTileProvider(dict.Dict{}, nil) if err == provider.ErrUnsupported { t.Fatalf("supported, expected any but unsupported got %v", err) } diff --git a/provider/gpkg/gpkg.go b/provider/gpkg/gpkg.go index fe369219f..65649ed07 100644 --- a/provider/gpkg/gpkg.go +++ b/provider/gpkg/gpkg.go @@ -76,7 +76,7 @@ func (p *Provider) Layers() ([]provider.LayerInfo, error) { return ls, nil } -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { log.Debugf("fetching layer %v", layer) pLayer := p.layers[layer] @@ -122,7 +122,7 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider // If layer was specified via "sql" in config, collect it z, _, _ := tile.ZXY() qtext = replaceTokens(pLayer.sql, z, tileBBox) - qtext = provider.ReplaceParams(queryParams, qtext, &args) + qtext = queryParams.ReplaceParams(qtext, &args) } log.Debugf("qtext: %v", qtext) diff --git a/provider/gpkg/gpkg_register.go b/provider/gpkg/gpkg_register.go index c1b4c52f3..791550fe9 100644 --- a/provider/gpkg/gpkg_register.go +++ b/provider/gpkg/gpkg_register.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -160,7 +161,6 @@ func extractColDefsFromSQL(sql string) []string { func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) { // this query is used to read the metadata from the gpkg_contents, gpkg_geometry_columns, and // sqlite_master tables for tables that store geographic features. - //goland:noinspection SqlResolve qtext := ` SELECT c.table_name, c.min_x, c.min_y, c.max_x, c.max_y, c.srs_id, gc.column_name, gc.geometry_type_name, sm.sql @@ -221,7 +221,7 @@ func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) return geomTableDetails, nil } -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { log.Debugf("config: %v", config) @@ -321,7 +321,6 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { return nil, fmt.Errorf("table %q does not exist", tablename) } - layer.tablename = tablename layer.tagFieldnames = tagFieldnames layer.geomFieldname = d.geomFieldname diff --git a/provider/gpkg/gpkg_register_internal_test.go b/provider/gpkg/gpkg_register_internal_test.go index c99b4b563..a0ba0b808 100644 --- a/provider/gpkg/gpkg_register_internal_test.go +++ b/provider/gpkg/gpkg_register_internal_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -1095,7 +1096,7 @@ func TestCleanup(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := NewTileProvider(tc.config) + _, err := NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return diff --git a/provider/gpkg/gpkg_test.go b/provider/gpkg/gpkg_test.go index 8941e74af..b3142dd13 100644 --- a/provider/gpkg/gpkg_test.go +++ b/provider/gpkg/gpkg_test.go @@ -184,7 +184,7 @@ func TestNewTileProvider(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("expectedErr %v got %v", tc.expectedErr, err) @@ -263,7 +263,7 @@ func TestTileFeatures(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("new tile, expected nil got %v", err) return @@ -411,7 +411,7 @@ func TestConfigs(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return @@ -593,7 +593,7 @@ func TestOpenNonExistantFile(t *testing.T) { os.Remove(NONEXISTANTFILE) fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := gpkg.NewTileProvider(tc.config) + _, err := gpkg.NewTileProvider(tc.config, nil) if reflect.TypeOf(err) != reflect.TypeOf(tc.err) { t.Errorf("expected error, expected %v got %v", tc.err, err) } @@ -601,13 +601,13 @@ func TestOpenNonExistantFile(t *testing.T) { } tests := map[string]tcase{ - "empty": tcase{ + "empty": { config: dict.Dict{ gpkg.ConfigKeyFilePath: "", }, err: gpkg.ErrInvalidFilePath{FilePath: ""}, }, - "nonexistance": tcase{ + "nonexistance": { // should not exists config: dict.Dict{ gpkg.ConfigKeyFilePath: NONEXISTANTFILE, diff --git a/provider/map.go b/provider/map.go new file mode 100644 index 000000000..d59ccd0b9 --- /dev/null +++ b/provider/map.go @@ -0,0 +1,14 @@ +package provider + +import "github.com/go-spatial/tegola/internal/env" + +// A Map represents a map in the Tegola Config file. +type Map struct { + Name env.String `toml:"name"` + Attribution env.String `toml:"attribution"` + Bounds []env.Float `toml:"bounds"` + Center [3]env.Float `toml:"center"` + Layers []MapLayer `toml:"layers"` + Parameters []QueryParameter `toml:"params"` + TileBuffer *env.Int `toml:"tile_buffer"` +} diff --git a/provider/map_layer.go b/provider/map_layer.go new file mode 100644 index 000000000..4a29d61a4 --- /dev/null +++ b/provider/map_layer.go @@ -0,0 +1,51 @@ +package provider + +import ( + "fmt" + "strings" + + "github.com/go-spatial/tegola/internal/env" +) + +// MapLayer represents a the config for a layer in a map +type MapLayer struct { + // Name is optional. If it's not defined the name of the ProviderLayer will be used. + // Name can also be used to group multiple ProviderLayers under the same namespace. + Name env.String `toml:"name"` + ProviderLayer env.String `toml:"provider_layer"` + MinZoom *env.Uint `toml:"min_zoom"` + MaxZoom *env.Uint `toml:"max_zoom"` + DefaultTags env.Dict `toml:"default_tags"` + // DontSimplify indicates whether feature simplification should be applied. + // We use a negative in the name so the default is to simplify + DontSimplify env.Bool `toml:"dont_simplify"` + // DontClip indicates whether feature clipping should be applied. + // We use a negative in the name so the default is to clipping + DontClip env.Bool `toml:"dont_clip"` + // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. + // We use a negative in the name so the default is to clean + DontClean env.Bool `toml:"dont_clean"` +} + +// ProviderLayerName returns the names of the layer and provider or an error +func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { + // split the provider layer (syntax is provider.layer) + plParts := strings.Split(string(ml.ProviderLayer), ".") + if len(plParts) != 2 { + // TODO (beymak): Properly handle the error + return "", "", fmt.Errorf("config: invalid provider layer name (%v)", ml.ProviderLayer) + // return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} + } + return plParts[0], plParts[1], nil +} + +// GetName will return the user-defined Layer name from the config, +// or if the name is empty, return the name of the layer associated with +// the provider +func (ml MapLayer) GetName() (string, error) { + if ml.Name != "" { + return string(ml.Name), nil + } + _, name, err := ml.ProviderLayerName() + return name, err +} diff --git a/provider/mvt_provider.go b/provider/mvt_provider.go index 558682a9f..4f192fec8 100644 --- a/provider/mvt_provider.go +++ b/provider/mvt_provider.go @@ -10,8 +10,8 @@ type MVTTiler interface { Layerer // MVTForLayers will return a MVT byte array or an error for the given layer names. - MVTForLayers(ctx context.Context, tile Tile, params map[string]QueryParameter, layers []Layer) ([]byte, error) + MVTForLayers(ctx context.Context, tile Tile, params Params, layers []Layer) ([]byte, error) } // MVTInitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type MVTInitFunc func(dicter dict.Dicter) (MVTTiler, error) +type MVTInitFunc func(dicter dict.Dicter, maps []Map) (MVTTiler, error) diff --git a/provider/paramater_decoders.go b/provider/paramater_decoders.go new file mode 100644 index 000000000..d1a2f7b91 --- /dev/null +++ b/provider/paramater_decoders.go @@ -0,0 +1,19 @@ +package provider + +import "strconv" + +// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters +var ParamTypeDecoders = map[string]func(string) (interface{}, error){ + "int": func(s string) (interface{}, error) { + return strconv.Atoi(s) + }, + "float": func(s string) (interface{}, error) { + return strconv.ParseFloat(s, 32) + }, + "string": func(s string) (interface{}, error) { + return s, nil + }, + "bool": func(s string) (interface{}, error) { + return strconv.ParseBool(s) + }, +} diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 1bd5889b4..8ec21bf32 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -171,6 +171,7 @@ const ( ) const ( + ConfigKeyName = "name" ConfigKeyURI = "uri" ConfigKeyHost = "host" ConfigKeyPort = "port" @@ -406,26 +407,24 @@ func BuildDBConfig(uri string) (*pgxpool.Config, error) { // trying to create a driver. This Provider supports the following fields // in the provided map[string]interface{} map: // -// host (string): [Required] postgis database host -// port (int): [Required] postgis database port (required) -// database (string): [Required] postgis database name -// user (string): [Required] postgis database user -// password (string): [Required] postgis database password -// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) -// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. -// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties +// name (string): [Required] name of the provider +// host (string): [Required] postgis database host +// port (int): [Required] postgis database port (required) +// database (string): [Required] postgis database name +// user (string): [Required] postgis database user +// password (string): [Required] postgis database password +// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) +// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. +// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties // -// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. -// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. -// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom -// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid -// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. -// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). -// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: -// -// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. -// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. -func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { +// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. +// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. +// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom +// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid +// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. +// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). +// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: +func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string) (*Provider, error) { uri, params, err := BuildURI(config) if err != nil { return nil, err @@ -453,7 +452,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) dbconfig, err := BuildDBConfig(uri.String()) if err != nil { - return nil, fmt.Errorf("failed while building db config: %v", err) + return nil, fmt.Errorf("failed while building db config: %w", err) } srid := DefaultSRID @@ -472,7 +471,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) pool, err := pgxpool.ConnectConfig(context.Background(), &p.config) if err != nil { - return nil, fmt.Errorf("failed while creating connection pool: %v", err) + return nil, fmt.Errorf("failed while creating connection pool: %w", err) } p.pool = &connectionPoolCollector{Pool: pool} @@ -488,7 +487,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) lName, err := layer.String(ConfigKeyLayerName, nil) if err != nil { - return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %v", i, err) + return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %w", i, err) } if j, ok := lyrsSeen[lName]; ok { @@ -605,7 +604,12 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) } } else { - if err = p.inspectLayerGeomType(&l); err != nil { + pname, err := config.String(ConfigKeyName, nil) + if err != nil { + return nil, err + } + + if err = p.inspectLayerGeomType(pname, &l, maps); err != nil { return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w\nif custom parameters are used, remember to set %s for the provider", l.name, err, ConfigKeyGeomType) } } @@ -698,7 +702,7 @@ func (p Provider) setLayerGeomType(l *Layer, geomType string) error { // inspectLayerGeomType sets the geomType field on the layer by running the SQL // and reading the geom type in the result set -func (p Provider) inspectLayerGeomType(l *Layer) error { +func (p Provider) inspectLayerGeomType(pname string, l *Layer, maps []provider.Map) error { var err error // we want to know the geom type instead of returning the geom data so we modify the SQL @@ -732,12 +736,20 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } - // remove all parameter tokens for inspection - // crossing our fingers that the query is still valid 🤞 - // if not, the user will have to specify `geometry_type` in the config - sql = stripParams(sql) + // substitute default values to parameter + params := extractQueryParamValues(pname, maps, l) + + args := make([]interface{}, 0) + sql = params.ReplaceParams(sql, &args) + + if provider.ParameterTokenRegexp.MatchString(sql) { + // remove all parameter tokens for inspection + // crossing our fingers that the query is still valid 🤞 + // if not, the user will have to specify `geometry_type` in the config + sql = provider.ParameterTokenRegexp.ReplaceAllString(sql, "") + } - rows, err := p.pool.Query(context.Background(), sql) + rows, err := p.pool.Query(context.Background(), sql, args...) if err != nil { return err } @@ -804,8 +816,7 @@ func (p Provider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures adheres to the provider.Tiler interface -// TODO (bemyak): Make an actual use of QueryParams -func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, params provider.Params, fn func(f *provider.Feature) error) error { var mapName string { @@ -828,7 +839,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // replace configured query parameters if any args := make([]interface{}, 0) - sql = provider.ReplaceParams(queryParams, sql, &args) + sql = params.ReplaceParams(sql, &args) if err != nil { return err } @@ -941,7 +952,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params map[string]provider.QueryParameter, layers []provider.Layer) ([]byte, error) { +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params provider.Params, layers []provider.Layer) ([]byte, error) { var ( err error sqls = make([]string, 0, len(layers)) @@ -977,7 +988,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params m } // replace configured query parameters if any - sql = provider.ReplaceParams(params, sql, &args) + sql = params.ReplaceParams(sql, &args) // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index bd1346bd0..bec0787b1 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -98,7 +98,8 @@ func TestMVTProviders(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - prvd, err := NewMVTTileProvider(config) + config[ConfigKeyName] = "provider_name" + prvd, err := NewMVTTileProvider(config, nil) // for now we will just check the length of the bytes. if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { @@ -168,7 +169,8 @@ func TestLayerGeomType(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - provider, err := NewTileProvider(config) + config[ConfigKeyName] = "provider_name" + provider, err := NewTileProvider(config, nil) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { t.Logf("error %#v", err) @@ -276,11 +278,11 @@ func TestLayerGeomType(t *testing.T) { TCConfig: TCConfig{ ConfigOverride: map[string]interface{}{ ConfigKeyURI: fmt.Sprintf("postgres://%v:%v@%v:%v/%v", - defaultEnvConfig["user"], - defaultEnvConfig["password"], - defaultEnvConfig["host"], - defaultEnvConfig["port"], - defaultEnvConfig["database"], + DefaultEnvConfig["user"], + DefaultEnvConfig["password"], + DefaultEnvConfig["host"], + DefaultEnvConfig["port"], + DefaultEnvConfig["database"], ), ConfigKeyHost: "", ConfigKeyPort: "", diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index 3907e1d90..22f4f3ffc 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -165,7 +165,8 @@ func TestNewTileProvider(t *testing.T) { fn := func(tc postgis.TCConfig) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - _, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + _, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unable to create a new provider. err: %v", err) return @@ -203,7 +204,8 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - p, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + p, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unexpected error; unable to create a new provider, expected: nil Got %v", err) return diff --git a/provider/postgis/register.go b/provider/postgis/register.go index ab0939be5..4c3a271b2 100644 --- a/provider/postgis/register.go +++ b/provider/postgis/register.go @@ -40,9 +40,9 @@ const ( // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { - return CreateProvider(config, ProviderType) +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { + return CreateProvider(config, maps, ProviderType) } -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { - return CreateProvider(config, MVTProviderType) +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { + return CreateProvider(config, maps, MVTProviderType) } diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 1d68c6b07..7dcb5df07 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -10,6 +10,7 @@ import ( "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/basic" "github.com/go-spatial/tegola/config" + "github.com/go-spatial/tegola/internal/env" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/provider" "github.com/jackc/pgproto3/v2" @@ -173,9 +174,25 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } -// stripParams will remove all parameter tokens from the query -func stripParams(sql string) string { - return provider.ParameterTokenRegexp.ReplaceAllString(sql, "") +// extractQueryParamValues finds default values for SQL tokens and constructs query parameter values out of them +func extractQueryParamValues(pname string, maps []provider.Map, layer *Layer) provider.Params { + result := make(provider.Params, 0) + + expectedMapName := fmt.Sprintf("%s.%s", pname, layer.name) + for _, m := range maps { + for _, l := range m.Layers { + if l.ProviderLayer == env.String(expectedMapName) { + for _, p := range m.Parameters { + pv, err := p.ToDefaultValue() + if err == nil { + result[p.Token] = pv + } + } + } + } + } + + return result } // uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index a662a047b..cf67d37b9 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -74,128 +74,6 @@ func TestReplaceTokens(t *testing.T) { } } -func TestReplaceParams(t *testing.T) { - type tcase struct { - params map[string]provider.QueryParameter - sql string - expectedSql string - expectedArgs []interface{} - } - - fn := func(tc tcase) func(t *testing.T) { - return func(t *testing.T) { - args := make([]interface{}, 0) - out := provider.ReplaceParams(tc.params, tc.sql, &args) - - if out != tc.expectedSql { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) - return - } - - if len(tc.expectedArgs) != len(args) { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - for i, arg := range tc.expectedArgs { - if arg != args[i] { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - } - } - } - - tests := map[string]tcase{ - "nil params": { - params: nil, - sql: "SELECT * FROM table", - expectedSql: "SELECT * FROM table", - expectedArgs: []interface{}{}, - }, - "int replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: 1, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{1}, - }, - "string replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: "test", - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{"test"}, - }, - "null replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: nil, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{nil}, - }, - "complex sql replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "WHERE PARAM=?", - Value: 1, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM=$1", - expectedArgs: []interface{}{1}, - }, - "subquery removal": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "", - Value: nil, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table ", - expectedArgs: []interface{}{}, - }, - "multiple params": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "???", - Value: 1, - }, - "!PARAM2!": { - Token: "!PARAM2!", - SQL: "???", - Value: 2, - }, - }, - sql: "!PARAM!!PARAM2!", - expectedSql: "$1$1$1$2$2$2", - expectedArgs: []interface{}{1, 2}, - }, - } - - for name, tc := range tests { - t.Run(name, fn(tc)) - } -} - func TestUppercaseTokens(t *testing.T) { type tcase struct { str string diff --git a/provider/provider.go b/provider/provider.go index 6fc2fe2c2..6813892cb 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "regexp" - "strings" "github.com/go-spatial/geom" "github.com/go-spatial/geom/slippy" @@ -109,59 +108,13 @@ type Tile interface { // ParameterTokenRegexp to validate QueryParameters var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") -// Query parameter holds normalized parameter data ready to be inserted in the -// final query -type QueryParameter struct { - // Token to replace e.g., !TOKEN! - Token string - // SQL expression to be inserted. Contains "?" that will be replaced with an - // ordinal argument e.g., "$1" - SQL string - // Value that will be passed to the final query in arguments list - Value interface{} - // Raw parameter values for debugging and monitoring - RawValues map[string]string -} - -// ReplaceParams substitutes configured query parameter tokens for their values -// within the provided SQL string -func ReplaceParams(params map[string]QueryParameter, sql string, args *[]interface{}) string { - if params == nil { - return sql - } - - for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { - param := params[token] - - // Replace every ? in the param's SQL with a positional argument - paramSQL := "" - argFound := false - for _, c := range param.SQL { - if c == '?' { - if !argFound { - *args = append(*args, param.Value) - argFound = true - } - paramSQL += fmt.Sprintf("$%d", len(*args)) - } else { - paramSQL += string(c) - } - } - - // Finally, replace current token with the prepared SQL - sql = strings.Replace(sql, token, paramSQL, 1) - } - - return sql -} - // Tiler is a Layers that allows one to encode features in that layer type Tiler interface { Layerer // TileFeature will stream decoded features to the callback function fn // if fn returns ErrCanceled, the TileFeatures method should stop processing - TileFeatures(ctx context.Context, layer string, t Tile, queryParams map[string]QueryParameter, fn func(f *Feature) error) error + TileFeatures(ctx context.Context, layer string, t Tile, params Params, fn func(f *Feature) error) error } // TilerUnion represents either a Std Tiler or and MVTTiler; only one should be not nil. @@ -183,7 +136,7 @@ func (tu TilerUnion) Layers() ([]LayerInfo, error) { } // InitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type InitFunc func(dicter dict.Dicter) (Tiler, error) +type InitFunc func(dicter dict.Dicter, maps []Map) (Tiler, error) // CleanupFunc is called to when the system is shutting down, this allows the provider to cleanup. type CleanupFunc func() @@ -280,7 +233,7 @@ func Drivers(types ...providerType) (l []string) { // For function returns a configure provider of the given type; The provider may be a mvt provider or // a std provider. The correct entry in TilerUnion will not be nil. If there is an error both entries // will be nil. -func For(name string, config dict.Dicter) (val TilerUnion, err error) { +func For(name string, config dict.Dicter, maps []Map) (val TilerUnion, err error) { var ( driversList = Drivers() ) @@ -292,11 +245,11 @@ func For(name string, config dict.Dicter) (val TilerUnion, err error) { return val, ErrUnknownProvider{KnownProviders: driversList, Name: name} } if p.init != nil { - val.Std, err = p.init(config) + val.Std, err = p.init(config, maps) return val, err } if p.mvtInit != nil { - val.Mvt, err = p.mvtInit(config) + val.Mvt, err = p.mvtInit(config, maps) return val, err } return val, ErrInvalidRegisteredProvider{Name: name} diff --git a/provider/provider_test.go b/provider/provider_test.go index 220ef94a4..d67a98a93 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -12,7 +12,7 @@ func TestProviderInterface(t *testing.T) { stdName = provider.TypeStd.Prefix() + test.Name mvtName = provider.TypeMvt.Prefix() + test.Name ) - if _, err := provider.For(stdName, nil); err != nil { + if _, err := provider.For(stdName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } @@ -23,7 +23,7 @@ func TestProviderInterface(t *testing.T) { if test.Count != 0 { t.Errorf(" expected count , expected 0 got %v", test.Count) } - if _, err := provider.For(mvtName, nil); err != nil { + if _, err := provider.For(mvtName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } diff --git a/provider/query_parameter.go b/provider/query_parameter.go new file mode 100644 index 000000000..181169bcb --- /dev/null +++ b/provider/query_parameter.go @@ -0,0 +1,65 @@ +package provider + +import ( + "fmt" + "strings" +) + +// QueryParameter represents an HTTP query parameter specified for use with +// a given map instance. +type QueryParameter struct { + Name string `toml:"name"` + Token string `toml:"token"` + Type string `toml:"type"` + SQL string `toml:"sql"` + // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or + // default_value can be specified + DefaultSQL string `toml:"default_sql"` + DefaultValue string `toml:"default_value"` +} + +// Normalize normalizes param and sets default values +func (param *QueryParameter) Normalize() { + param.Token = strings.ToUpper(param.Token) + + if len(param.SQL) == 0 { + param.SQL = "?" + } +} + +func (param *QueryParameter) ToValue(rawValue string) (QueryParameterValue, error) { + val, err := ParamTypeDecoders[param.Type](rawValue) + if err != nil { + return QueryParameterValue{}, err + } + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: rawValue, + }, nil +} + +func (param *QueryParameter) ToDefaultValue() (QueryParameterValue, error) { + if len(param.DefaultValue) > 0 { + val, err := ParamTypeDecoders[param.Type](param.DefaultValue) + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: "", + }, err + } + if len(param.DefaultSQL) > 0 { + return QueryParameterValue{ + Token: param.Token, + SQL: param.DefaultSQL, + Value: nil, + RawParam: param.Name, + RawValue: "", + }, nil + } + return QueryParameterValue{}, fmt.Errorf("the required parameter %s is not specified", param.Name) +} diff --git a/provider/query_parameter_value.go b/provider/query_parameter_value.go new file mode 100644 index 000000000..10fec3170 --- /dev/null +++ b/provider/query_parameter_value.go @@ -0,0 +1,76 @@ +package provider + +import ( + "fmt" + "strings" +) + +// Query parameter holds normalized parameter data ready to be inserted in the +// final query +type QueryParameterValue struct { + // Token to replace e.g., !TOKEN! + Token string + // SQL expression to be inserted. Contains "?" that will be replaced with an + // ordinal argument e.g., "$1" + SQL string + // Value that will be passed to the final query in arguments list + Value interface{} + // Raw parameter and value for debugging and monitoring + RawParam string + // RawValue will be "" if the param wasn't passed and defaults were used + RawValue string +} + +type Params map[string]QueryParameterValue + +// ReplaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func (params Params) ReplaceParams(sql string, args *[]interface{}) string { + if params == nil { + return sql + } + + var ( + cache = make(map[string]string) + sb strings.Builder + ) + + for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { + resultSQL, ok := cache[token] + if ok { + // Already have it cached, replace the token and move on. + sql = strings.ReplaceAll(sql, token, resultSQL) + continue + } + + param, ok := params[token] + if !ok { + // Unknown token, ignoring + continue + } + + sb.Reset() + sb.Grow(len(param.SQL)) + argFound := false + + // Replace every `?` in the param's SQL with a positional argument + for _, c := range param.SQL { + if c != '?' { + sb.WriteRune(c) + continue + } + + if !argFound { + *args = append(*args, param.Value) + argFound = true + } + sb.WriteString(fmt.Sprintf("$%d", len(*args))) + } + + resultSQL = sb.String() + cache[token] = resultSQL + sql = strings.ReplaceAll(sql, token, resultSQL) + } + + return sql +} diff --git a/provider/query_parameter_value_test.go b/provider/query_parameter_value_test.go new file mode 100644 index 000000000..204bca97f --- /dev/null +++ b/provider/query_parameter_value_test.go @@ -0,0 +1,137 @@ +package provider + +import "testing" + +func TestReplaceParams(t *testing.T) { + type tcase struct { + params Params + sql string + expectedSql string + expectedArgs []interface{} + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + args := make([]interface{}, 0) + out := tc.params.ReplaceParams(tc.sql, &args) + + if out != tc.expectedSql { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) + return + } + + if len(tc.expectedArgs) != len(args) { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + for i, arg := range tc.expectedArgs { + if arg != args[i] { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + } + } + } + + tests := map[string]tcase{ + "nil params": { + params: nil, + sql: "SELECT * FROM table", + expectedSql: "SELECT * FROM table", + expectedArgs: []interface{}{}, + }, + "int replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{1}, + }, + "string replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: "test", + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{"test"}, + }, + "null replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: nil, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{nil}, + }, + "complex sql replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "WHERE param=?", + Value: 1, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table WHERE param=$1", + expectedArgs: []interface{}{1}, + }, + "subquery removal": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "", + Value: nil, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table ", + expectedArgs: []interface{}{}, + }, + "multiple params": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "? ? ?", + Value: 1, + }, + "!PARAM2!": { + Token: "!PARAM2!", + SQL: "???", + Value: 2, + }, + }, + sql: "!PARAM! !PARAM2! !PARAM!", + expectedSql: "$1 $1 $1 $2$2$2 $1 $1 $1", + expectedArgs: []interface{}{1, 2}, + }, + "unknown token": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "!NOT_PARAM! !PARAM! !NOT_PARAM!", + expectedSql: "!NOT_PARAM! $1 !NOT_PARAM!", + expectedArgs: []interface{}{1}, + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/provider/test/emptycollection/provider.go b/provider/test/emptycollection/provider.go index 458a4e540..eef8d9ac0 100644 --- a/provider/test/emptycollection/provider.go +++ b/provider/test/emptycollection/provider.go @@ -19,7 +19,7 @@ func init() { } // NewProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { Count++ return &TileProvider{}, nil } @@ -40,7 +40,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TilFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box _, srid := t.Extent() diff --git a/provider/test/provider.go b/provider/test/provider.go index ca0942693..011242afb 100644 --- a/provider/test/provider.go +++ b/provider/test/provider.go @@ -28,7 +28,7 @@ func init() { } // NewTileProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { lock.Lock() Count++ lock.Unlock() @@ -37,7 +37,7 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // NewMVTTileProvider setups a test provider for mvt tiles providers. The only supported parameter is // "test_file", which should point to a mvt tile file to return for MVTForLayers -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { lock.Lock() MVTCount++ lock.Unlock() @@ -86,7 +86,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := t.Extent() @@ -103,7 +103,7 @@ func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provid } // MVTForLayers mocks out MVTForLayers by just returning the MVTTile bytes, this will never error -func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ map[string]provider.QueryParameter, _ []provider.Layer) ([]byte, error) { +func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ provider.Params, _ []provider.Layer) ([]byte, error) { // TODO(gdey): fill this out. if tp == nil { return nil, nil diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index 9cfbfa65e..39345a3b0 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/go-spatial/tegola/provider" @@ -101,9 +100,11 @@ func (req *HandleMapLayerZXY) parseURI(r *http.Request) error { // map_name - map name in the config file // layer_name - name of the single map layer to render // z, x, y - tile coordinates as described in the Slippy Map Tilenames specification -// z - zoom level -// x - row -// y - column +// +// z - zoom level +// x - row +// y - column +// // param - configurable query parameters and their values func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { // parse our URI @@ -162,7 +163,7 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // check for query parameters and populate param map with their values - params, err := req.extractParameters(r) + params, err := extractParameters(m, r) if err != nil { log.Error(err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -206,44 +207,28 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (req *HandleMapLayerZXY) extractParameters(r *http.Request) (map[string]provider.QueryParameter, error) { - var params map[string]provider.QueryParameter - if req.Atlas.HasParams(req.mapName) { - params = make(map[string]provider.QueryParameter) +func extractParameters(m atlas.Map, r *http.Request) (provider.Params, error) { + var params provider.Params + if m.Params != nil && len(m.Params) > 0 { + params = make(provider.Params) err := r.ParseForm() if err != nil { return nil, err } - for _, param := range req.Atlas.GetParams(req.mapName) { + for _, param := range m.Params { if r.Form.Has(param.Name) { - val, err := config.ParamTypeDecoders[param.Type](r.Form.Get(param.Name)) + val, err := param.ToValue(r.Form.Get(param.Name)) if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultValue) > 0 { - val, err := config.ParamTypeDecoders[param.Type](param.DefaultValue) + params[param.Token] = val + } else { + p, err := param.ToDefaultValue() if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultSQL) > 0 { - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.DefaultSQL, - Value: nil, - } - } else { - return nil, fmt.Errorf("the required parameter %s is not specified", param.Name) + params[param.Token] = p } } } diff --git a/tile.go b/tile.go index 609238f6f..38ecd7bc5 100644 --- a/tile.go +++ b/tile.go @@ -17,7 +17,7 @@ const ( var UnknownConversionError = fmt.Errorf("do not know how to convert value to requested value") -//Tile slippy map tilenames +// Tile slippy map tilenames // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames type Tile struct { Z uint @@ -198,7 +198,7 @@ func (t *Tile) ZLevel() uint { return t.Z } -//ZRes takes a web mercator zoom level and returns the pixel resolution for that +// ZRes takes a web mercator zoom level and returns the pixel resolution for that // scale, assuming t.Extent x t.Extent pixel tiles. Non-integer zoom levels are accepted. // ported from: https://raw.githubusercontent.com/mapbox/postgis-vt-util/master/postgis-vt-util.sql // 40075016.6855785 is the equator in meters for WGS84 at z=0