diff --git a/atlas/map.go b/atlas/map.go index 5f1385c8b..aab8e79de 100644 --- a/atlas/map.go +++ b/atlas/map.go @@ -9,7 +9,6 @@ import ( "strings" "sync" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/golang/protobuf/proto" @@ -56,7 +55,7 @@ type Map struct { Center [3]float64 Layers []Layer // Params holds configured query parameters - Params []config.QueryParameter + Params []provider.QueryParameter SRID uint64 // MVT output values @@ -120,7 +119,7 @@ func (m Map) AddDebugLayers() Map { m.Layers = layers // setup a debug provider - debugProvider, _ := debug.NewTileProvider(dict.Dict{}) + debugProvider, _ := debug.NewTileProvider(dict.Dict{}, nil) m.Layers = append(layers, []Layer{ { @@ -183,7 +182,7 @@ func (m Map) FilterLayersByName(names ...string) Map { return m } -func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // get the list of our layers ptile := provider.NewTile(tile.Z, tile.X, tile.Y, uint(m.TileBuffer), uint(m.SRID)) @@ -200,7 +199,7 @@ func (m Map) encodeMVTProviderTile(ctx context.Context, tile *slippy.Tile, param // encodeMVTTile will encode the given tile into mvt format // TODO (arolek): support for max zoom -func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { // tile container var mvtTile mvt.Tile @@ -380,7 +379,7 @@ func (m Map) encodeMVTTile(ctx context.Context, tile *slippy.Tile, params map[st } // Encode will encode the given tile into mvt format -func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params map[string]provider.QueryParameter) ([]byte, error) { +func (m Map) Encode(ctx context.Context, tile *slippy.Tile, params provider.Params) ([]byte, error) { var ( tileBytes []byte err error diff --git a/cmd/internal/register/maps.go b/cmd/internal/register/maps.go index 69dec943e..bb4e62c9f 100644 --- a/cmd/internal/register/maps.go +++ b/cmd/internal/register/maps.go @@ -11,7 +11,7 @@ import ( "github.com/go-spatial/tegola/provider" ) -func webMercatorMapFromConfigMap(cfg config.Map) (newMap atlas.Map) { +func webMercatorMapFromConfigMap(cfg provider.Map) (newMap atlas.Map) { newMap = atlas.NewWebMercatorMap(string(cfg.Name)) newMap.Attribution = SanitizeAttribution(string(cfg.Attribution)) newMap.Params = cfg.Parameters @@ -47,7 +47,7 @@ func layerInfosFindByName(infos []provider.LayerInfo, name string) provider.Laye return nil } -func atlasLayerFromConfigLayer(cfg *config.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { +func atlasLayerFromConfigLayer(cfg *provider.MapLayer, mapName string, layerProvider provider.Layerer) (layer atlas.Layer, err error) { var ( // providerLayer is primary used for error reporting. providerLayer = string(cfg.ProviderLayer) @@ -124,7 +124,7 @@ func selectProvider(name string, mapName string, newMap *atlas.Map, providers ma } // Maps registers maps with with atlas -func Maps(a *atlas.Atlas, maps []config.Map, providers map[string]provider.TilerUnion) error { +func Maps(a *atlas.Atlas, maps []provider.Map, providers map[string]provider.TilerUnion) error { var ( layerer provider.Layerer diff --git a/cmd/internal/register/maps_test.go b/cmd/internal/register/maps_test.go index abe05dcc1..5573bb8c4 100644 --- a/cmd/internal/register/maps_test.go +++ b/cmd/internal/register/maps_test.go @@ -6,15 +6,15 @@ import ( "github.com/go-spatial/tegola/atlas" "github.com/go-spatial/tegola/cmd/internal/register" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/dict" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" ) func TestMaps(t *testing.T) { type tcase struct { atlas atlas.Atlas - maps []config.Map + maps []provider.Map providers []dict.Dict expectedErr error } @@ -29,7 +29,7 @@ func TestMaps(t *testing.T) { provArr[i] = tc.providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, tc.maps) if err != nil { t.Errorf("unexpected err: %v", err) return @@ -45,10 +45,10 @@ func TestMaps(t *testing.T) { tests := map[string]tcase{ "provider layer invalid": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar", }, @@ -67,10 +67,10 @@ func TestMaps(t *testing.T) { }, }, "provider not found": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bar.baz", }, @@ -82,10 +82,10 @@ func TestMaps(t *testing.T) { }, }, "provider layer not registered with provider": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.bar", }, @@ -105,10 +105,10 @@ func TestMaps(t *testing.T) { }, }, "default tags": { - maps: []config.Map{ + maps: []provider.Map{ { Name: "foo", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "test.debug-tile-outline", DefaultTags: env.Dict{ @@ -126,7 +126,7 @@ func TestMaps(t *testing.T) { }, }, "success": { - maps: []config.Map{}, + maps: []provider.Map{}, providers: []dict.Dict{ { "name": "test", diff --git a/cmd/internal/register/providers.go b/cmd/internal/register/providers.go index 1eb16faa9..4553b9fc1 100644 --- a/cmd/internal/register/providers.go +++ b/cmd/internal/register/providers.go @@ -33,7 +33,7 @@ func (e ErrProviderTypeInvalid) Error() string { } // Providers registers data provider backends -func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) { +func Providers(providers []dict.Dicter, maps []provider.Map) (map[string]provider.TilerUnion, error) { // holder for registered providers registeredProviders := map[string]provider.TilerUnion{} @@ -72,7 +72,7 @@ func Providers(providers []dict.Dicter) (map[string]provider.TilerUnion, error) } // register the provider - prov, err := provider.For(ptype, p) + prov, err := provider.For(ptype, p, maps) if err != nil { return registeredProviders, err } diff --git a/cmd/internal/register/providers_test.go b/cmd/internal/register/providers_test.go index c1f62a45b..9f8d44410 100644 --- a/cmd/internal/register/providers_test.go +++ b/cmd/internal/register/providers_test.go @@ -23,7 +23,7 @@ func TestProviders(t *testing.T) { provArr[i] = tc.config[i] } - _, err = register.Providers(provArr) + _, err = register.Providers(provArr, nil) if tc.expectedErr != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("invalid error. expected: %v, got %v", tc.expectedErr, err.Error()) diff --git a/cmd/tegola/cmd/root.go b/cmd/tegola/cmd/root.go index 3e41e8c34..177638ae2 100644 --- a/cmd/tegola/cmd/root.go +++ b/cmd/tegola/cmd/root.go @@ -121,7 +121,7 @@ func initConfig(configFile string, cacheRequired bool, logLevel string, logger s provArr[i] = conf.Providers[i] } - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, conf.Maps) if err != nil { return fmt.Errorf("could not register providers: %v", err) } diff --git a/cmd/tegola_lambda/main.go b/cmd/tegola_lambda/main.go index 4d1e19f4a..cf96b11ab 100644 --- a/cmd/tegola_lambda/main.go +++ b/cmd/tegola_lambda/main.go @@ -66,7 +66,7 @@ func init() { } // register the providers - providers, err := register.Providers(provArr) + providers, err := register.Providers(provArr, nil) if err != nil { log.Fatal(err) } diff --git a/config/config.go b/config/config.go index 09685c755..e4dd0ca48 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "os" - "strconv" "strings" "time" @@ -46,22 +45,6 @@ var ReservedTokens = map[string]struct{}{ GeomTypeToken: {}, } -// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters -var ParamTypeDecoders = map[string]func(string) (interface{}, error){ - "int": func(s string) (interface{}, error) { - return strconv.Atoi(s) - }, - "float": func(s string) (interface{}, error) { - return strconv.ParseFloat(s, 32) - }, - "string": func(s string) (interface{}, error) { - return s, nil - }, - "bool": func(s string) (interface{}, error) { - return strconv.ParseBool(s) - }, -} - var blacklistHeaders = []string{"content-encoding", "content-length", "content-type"} // Config represents a tegola config file. @@ -82,8 +65,8 @@ type Config struct { // 2. type -- this is the name the provider modules register // themselves under. (e.g. postgis, gpkg, mvt_postgis ) // Note: Use the type to figure out if the provider is a mvt or std provider - Providers []env.Dict `toml:"providers"` - Maps []Map `toml:"maps"` + Providers []env.Dict `toml:"providers"` + Maps []provider.Map `toml:"maps"` } // Webserver represents the config options for the webserver part of Tegola @@ -96,47 +79,36 @@ type Webserver struct { SSLKey env.String `toml:"ssl_key"` } -// A Map represents a map in the Tegola Config file. -type Map struct { - Name env.String `toml:"name"` - Attribution env.String `toml:"attribution"` - Bounds []env.Float `toml:"bounds"` - Center [3]env.Float `toml:"center"` - Layers []MapLayer `toml:"layers"` - Parameters []QueryParameter `toml:"params"` - TileBuffer *env.Int `toml:"tile_buffer"` -} - -// ValidateParams ensures configured params don't conflict with existing +// ValidateAndRegisterParams ensures configured params don't conflict with existing // query tokens or have overlapping names -func (m Map) ValidateParams() error { - if len(m.Parameters) == 0 { +func ValidateAndRegisterParams(mapName string, params []provider.QueryParameter) error { + if len(params) == 0 { return nil } usedNames := make(map[string]struct{}) usedTokens := make(map[string]struct{}) - for _, param := range m.Parameters { - if _, ok := ParamTypeDecoders[param.Type]; !ok { + for _, param := range params { + if _, ok := provider.ParamTypeDecoders[param.Type]; !ok { return ErrParamUnknownType{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultSQL) > 0 && len(param.DefaultValue) > 0 { return ErrParamTwoDefaults{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if len(param.DefaultValue) > 0 { - decoderFn := ParamTypeDecoders[param.Type] + decoderFn := provider.ParamTypeDecoders[param.Type] if _, err := decoderFn(param.DefaultValue); err != nil { return ErrParamInvalidDefault{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } @@ -144,28 +116,28 @@ func (m Map) ValidateParams() error { if _, ok := ReservedTokens[param.Token]; ok { return ErrParamTokenReserved{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if !provider.ParameterTokenRegexp.MatchString(param.Token) { return ErrParamBadTokenName{ - MapName: string(m.Name), + MapName: string(mapName), Parameter: param, } } if _, ok := usedNames[param.Name]; ok { - return ErrParamNameDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateName{ + MapName: string(mapName), Parameter: param, } } if _, ok := usedTokens[param.Token]; ok { - return ErrParamTokenDuplicate{ - MapName: string(m.Name), + return ErrParamDuplicateToken{ + MapName: string(mapName), Parameter: param, } } @@ -182,69 +154,6 @@ func (m Map) ValidateParams() error { return nil } -// MapLayer represents a the config for a layer in a map -type MapLayer struct { - // Name is optional. If it's not defined the name of the ProviderLayer will be used. - // Name can also be used to group multiple ProviderLayers under the same namespace. - Name env.String `toml:"name"` - ProviderLayer env.String `toml:"provider_layer"` - MinZoom *env.Uint `toml:"min_zoom"` - MaxZoom *env.Uint `toml:"max_zoom"` - DefaultTags env.Dict `toml:"default_tags"` - // DontSimplify indicates whether feature simplification should be applied. - // We use a negative in the name so the default is to simplify - DontSimplify env.Bool `toml:"dont_simplify"` - // DontClip indicates whether feature clipping should be applied. - // We use a negative in the name so the default is to clipping - DontClip env.Bool `toml:"dont_clip"` - // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. - // We use a negative in the name so the default is to clean - DontClean env.Bool `toml:"dont_clean"` -} - -// ProviderLayerName returns the names of the layer and provider or an error -func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { - // split the provider layer (syntax is provider.layer) - plParts := strings.Split(string(ml.ProviderLayer), ".") - if len(plParts) != 2 { - return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} - } - return plParts[0], plParts[1], nil -} - -// GetName will return the user-defined Layer name from the config, -// or if the name is empty, return the name of the layer associated with -// the provider -func (ml MapLayer) GetName() (string, error) { - if ml.Name != "" { - return string(ml.Name), nil - } - _, name, err := ml.ProviderLayerName() - return name, err -} - -// QueryParameter represents an HTTP query parameter specified for use with -// a given map instance. -type QueryParameter struct { - Name string `toml:"name"` - Token string `toml:"token"` - Type string `toml:"type"` - SQL string `toml:"sql"` - // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or - // default_value can be specified - DefaultSQL string `toml:"default_sql"` - DefaultValue string `toml:"default_value"` -} - -// Normalize normalizes param and sets default values -func (param *QueryParameter) Normalize() { - param.Token = strings.ToUpper(param.Token) - - if len(param.SQL) == 0 { - param.SQL = "?" - } -} - // Validate checks the config for issues func (c *Config) Validate() error { @@ -286,16 +195,16 @@ func (c *Config) Validate() error { } // check for map layer name / zoom collisions // map of layers to providers - mapLayers := map[string]map[string]MapLayer{} + mapLayers := map[string]map[string]provider.MapLayer{} for mapKey, m := range c.Maps { // validate any declared query parameters - if err := m.ValidateParams(); err != nil { + if err := ValidateAndRegisterParams(string(m.Name), m.Parameters); err != nil { return err } if _, ok := mapLayers[string(m.Name)]; !ok { - mapLayers[string(m.Name)] = map[string]MapLayer{} + mapLayers[string(m.Name)] = map[string]provider.MapLayer{} } // Set current provider to empty, for MVT providers diff --git a/config/config_test.go b/config/config_test.go index 8ae0f59e9..5bf34ced6 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/internal/env" + "github.com/go-spatial/tegola/provider" _ "github.com/go-spatial/tegola/provider/debug" _ "github.com/go-spatial/tegola/provider/postgis" _ "github.com/go-spatial/tegola/provider/test" @@ -167,14 +168,14 @@ func TestParse(t *testing.T) { name = "param2" token = "!PARAM2!" type = "int" - sql = "AND ANSWER = ?" + sql = "AND answer = ?" default_value = "42" [[maps.params]] name = "param3" token = "!PARAM3!" type = "float" - default_sql = "AND PI = 3.1415926" + default_sql = "AND pi = 3.1415926" `, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(12)), @@ -210,14 +211,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(12)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -227,7 +228,7 @@ func TestParse(t *testing.T) { DontClean: true, }, }, - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param1", Token: "!PARAM1!", @@ -238,7 +239,7 @@ func TestParse(t *testing.T) { Name: "param2", Token: "!PARAM2!", Type: "int", - SQL: "AND ANSWER = ?", + SQL: "AND answer = ?", DefaultValue: "42", }, { @@ -246,7 +247,7 @@ func TestParse(t *testing.T) { Token: "!PARAM3!", Type: "float", SQL: "?", - DefaultSQL: "AND PI = 3.1415926", + DefaultSQL: "AND pi = 3.1415926", }, }, }, @@ -360,14 +361,14 @@ func TestParse(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{ENV_TEST_CENTER_X, ENV_TEST_CENTER_Y, ENV_TEST_CENTER_Z}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: ENV_TEST_PROVIDER_LAYER, @@ -388,7 +389,7 @@ func TestParse(t *testing.T) { Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, TileBuffer: env.IntPtr(env.Int(64)), - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -542,13 +543,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: nil, @@ -586,13 +587,13 @@ func TestValidateMutateZoom(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(0), @@ -672,13 +673,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -736,13 +737,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { Name: "water", ProviderLayer: "provider1.water_0_5", @@ -806,13 +807,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -830,7 +831,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", MinZoom: env.UintPtr(10), @@ -889,13 +890,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water", }, @@ -906,7 +907,7 @@ func TestValidate(t *testing.T) { Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider2.water", }, @@ -958,13 +959,13 @@ func TestValidate(t *testing.T) { }, }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", Attribution: "Test Attribution", Bounds: []env.Float{-180, -85.05112877980659, 180, 85.0511287798066}, Center: [3]env.Float{-76.275329586789, 39.153492567373, 8.0}, - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1076,11 +1077,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1097,11 +1098,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1125,11 +1126,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1153,11 +1154,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "happy", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "bad.water_default_z", }, @@ -1182,11 +1183,11 @@ func TestValidate(t *testing.T) { "type": "test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "provider1.water_default_z", }, @@ -1214,11 +1215,11 @@ func TestValidate(t *testing.T) { "type": "mvt_test", }, }, - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "comingle", Attribution: "Test Attribution", - Layers: []config.MapLayer{ + Layers: []provider.MapLayer{ { ProviderLayer: "stdprovider1.water_default_z", }, @@ -1230,12 +1231,12 @@ func TestValidate(t *testing.T) { }, }, }, - "13 reserved parameter token": { + "13 reserved token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "bad_param", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1247,7 +1248,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTokenReserved{ MapName: "bad_param", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1256,10 +1257,10 @@ func TestValidate(t *testing.T) { }, "13 duplicate parameter name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_name", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1274,21 +1275,21 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamNameDuplicate{ + expectedErr: config.ErrParamDuplicateName{ MapName: "dupe_param_name", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!PARAM2!", Type: "int", }, }, }, - "13 duplicate parameter token": { + "13 duplicate token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "dupe_param_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!PARAM!", @@ -1303,9 +1304,9 @@ func TestValidate(t *testing.T) { }, }, }, - expectedErr: config.ErrParamTokenDuplicate{ + expectedErr: config.ErrParamDuplicateToken{ MapName: "dupe_param_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param2", Token: "!PARAM!", Type: "int", @@ -1314,10 +1315,10 @@ func TestValidate(t *testing.T) { }, "13 parameter unknown type": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_param_type", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1329,7 +1330,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamUnknownType{ MapName: "unknown_param_type", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "foo", @@ -1338,10 +1339,10 @@ func TestValidate(t *testing.T) { }, "13 parameter two defaults": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "unknown_two_defaults", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1355,7 +1356,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamTwoDefaults{ MapName: "unknown_two_defaults", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "string", @@ -1366,11 +1367,11 @@ func TestValidate(t *testing.T) { }, "13 parameter invalid default": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_default", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!BBOX!", @@ -1383,7 +1384,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamInvalidDefault{ MapName: "parameter_invalid_default", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!BBOX!", Type: "int", @@ -1393,10 +1394,10 @@ func TestValidate(t *testing.T) { }, "13 invalid token name": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "parameter_invalid_token", - Parameters: []config.QueryParameter{ + Parameters: []provider.QueryParameter{ { Name: "param", Token: "!Token with spaces!", @@ -1408,7 +1409,7 @@ func TestValidate(t *testing.T) { }, expectedErr: config.ErrParamBadTokenName{ MapName: "parameter_invalid_token", - Parameter: config.QueryParameter{ + Parameter: provider.QueryParameter{ Name: "param", Token: "!Token with spaces!", Type: "int", @@ -1444,14 +1445,14 @@ func TestConfigureTileBuffers(t *testing.T) { tests := map[string]tcase{ "1 tilebuffer is not set": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(64)), @@ -1462,7 +1463,7 @@ func TestConfigureTileBuffers(t *testing.T) { "2 tilebuffer is set in global section": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", }, @@ -1473,7 +1474,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(32)), @@ -1487,7 +1488,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, "3 tilebuffer is set in map section": { config: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1499,7 +1500,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, }, expected: config.Config{ - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1514,7 +1515,7 @@ func TestConfigureTileBuffers(t *testing.T) { "4 tilebuffer is set in global and map sections": { config: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), @@ -1523,7 +1524,7 @@ func TestConfigureTileBuffers(t *testing.T) { }, expected: config.Config{ TileBuffer: env.IntPtr(env.Int(32)), - Maps: []config.Map{ + Maps: []provider.Map{ { Name: "osm", TileBuffer: env.IntPtr(env.Int(16)), diff --git a/config/errors.go b/config/errors.go index be6e5a59f..ded68ddfb 100644 --- a/config/errors.go +++ b/config/errors.go @@ -3,6 +3,8 @@ package config import ( "fmt" "strings" + + "github.com/go-spatial/tegola/provider" ) type ErrMapNotFound struct { @@ -15,78 +17,76 @@ func (e ErrMapNotFound) Error() string { type ErrParamTokenReserved struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTokenReserved) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing reserved token %s", + return fmt.Sprintf("config: map %s parameter %s uses a reserved token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } -type ErrParamNameDuplicate struct { +type ErrParamDuplicateName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamNameDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares duplicate parameter with name %s", +func (e ErrParamDuplicateName) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a name already used by another parameter", e.MapName, e.Parameter.Name) } -type ErrParamTokenDuplicate struct { +type ErrParamDuplicateToken struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } -func (e ErrParamTokenDuplicate) Error() string { - return fmt.Sprintf("config: map %s redeclares existing parameter token %s in param %s", +func (e ErrParamDuplicateToken) Error() string { + return fmt.Sprintf("config: map %s parameter %s has a token name %s already used by another parameter", e.MapName, e.Parameter.Token, e.Parameter.Name) } type ErrParamUnknownType struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamUnknownType) Error() string { - validTypes := make([]string, len(ParamTypeDecoders)) - i := 0 - for k := range ParamTypeDecoders { - validTypes[i] = k - i++ + validTypes := make([]string, 0, len(provider.ParamTypeDecoders)) + for k := range provider.ParamTypeDecoders { + validTypes = append(validTypes, k) } - return fmt.Sprintf("config: map %s has type %s in param %s, which is not one of the known types: %s", - e.MapName, e.Parameter.Type, e.Parameter.Name, strings.Join(validTypes, ",")) + return fmt.Sprintf("config: map %s parameter %s has an unknown type %s, must be one of: %s", + e.MapName, e.Parameter.Name, e.Parameter.Type, strings.Join(validTypes, ",")) } type ErrParamTwoDefaults struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamTwoDefaults) Error() string { - return fmt.Sprintf("config: map %s has both default_value and default_sql defined in param %s", + return fmt.Sprintf("config: map %s parameter %s has both default_value and default_sql", e.MapName, e.Parameter.Name) } type ErrParamInvalidDefault struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamInvalidDefault) Error() string { - return fmt.Sprintf("config: map %s has default value in param %s that doesn't match the parameter's type %s", + return fmt.Sprintf("config: map %s parameter %s has a default value that is invalid for type %s", e.MapName, e.Parameter.Name, e.Parameter.Type) } type ErrParamBadTokenName struct { MapName string - Parameter QueryParameter + Parameter provider.QueryParameter } func (e ErrParamBadTokenName) Error() string { - return fmt.Sprintf("config: map %s has parameter %s referencing token with an invalid name %s", + return fmt.Sprintf("config: map %s parameter %s has an invalid token name %s", e.MapName, e.Parameter.Name, e.Parameter.Token) } diff --git a/provider/debug/debug.go b/provider/debug/debug.go index 212eab06e..ba96f41f1 100644 --- a/provider/debug/debug.go +++ b/provider/debug/debug.go @@ -26,25 +26,21 @@ func init() { } // NewProvider Setups a debug provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { return &Provider{}, nil } // Provider provides the debug provider type Provider struct{} -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := tile.Extent() - params := make([]string, len(queryParams)) - i := 0 + params := make([]string, 0, len(queryParams)) for _, param := range queryParams { - for k, v := range param.RawValues { - params[i] = fmt.Sprintf("%s=%s", k, v) - i++ - } + params = append(params, fmt.Sprintf("%s=%s", param.RawParam, param.RawValue)) } paramsStr := strings.Join(params, " ") diff --git a/provider/gpkg/cgo_test.go b/provider/gpkg/cgo_test.go index 8a64fb9b1..9d7f7902e 100644 --- a/provider/gpkg/cgo_test.go +++ b/provider/gpkg/cgo_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -11,7 +12,7 @@ import ( // This is a test to just see that the init function is doing something. func TestNewProviderStartup(t *testing.T) { - _, err := NewTileProvider(dict.Dict{}) + _, err := NewTileProvider(dict.Dict{}, nil) if err == provider.ErrUnsupported { t.Fatalf("supported, expected any but unsupported got %v", err) } diff --git a/provider/gpkg/gpkg.go b/provider/gpkg/gpkg.go index fe369219f..65649ed07 100644 --- a/provider/gpkg/gpkg.go +++ b/provider/gpkg/gpkg.go @@ -76,7 +76,7 @@ func (p *Provider) Layers() ([]provider.LayerInfo, error) { return ls, nil } -func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { log.Debugf("fetching layer %v", layer) pLayer := p.layers[layer] @@ -122,7 +122,7 @@ func (p *Provider) TileFeatures(ctx context.Context, layer string, tile provider // If layer was specified via "sql" in config, collect it z, _, _ := tile.ZXY() qtext = replaceTokens(pLayer.sql, z, tileBBox) - qtext = provider.ReplaceParams(queryParams, qtext, &args) + qtext = queryParams.ReplaceParams(qtext, &args) } log.Debugf("qtext: %v", qtext) diff --git a/provider/gpkg/gpkg_register.go b/provider/gpkg/gpkg_register.go index c1b4c52f3..791550fe9 100644 --- a/provider/gpkg/gpkg_register.go +++ b/provider/gpkg/gpkg_register.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -160,7 +161,6 @@ func extractColDefsFromSQL(sql string) []string { func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) { // this query is used to read the metadata from the gpkg_contents, gpkg_geometry_columns, and // sqlite_master tables for tables that store geographic features. - //goland:noinspection SqlResolve qtext := ` SELECT c.table_name, c.min_x, c.min_y, c.max_x, c.max_y, c.srs_id, gc.column_name, gc.geometry_type_name, sm.sql @@ -221,7 +221,7 @@ func featureTableMetaData(gpkg *sql.DB) (map[string]featureTableDetails, error) return geomTableDetails, nil } -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { log.Debugf("config: %v", config) @@ -321,7 +321,6 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { return nil, fmt.Errorf("table %q does not exist", tablename) } - layer.tablename = tablename layer.tagFieldnames = tagFieldnames layer.geomFieldname = d.geomFieldname diff --git a/provider/gpkg/gpkg_register_internal_test.go b/provider/gpkg/gpkg_register_internal_test.go index c99b4b563..a0ba0b808 100644 --- a/provider/gpkg/gpkg_register_internal_test.go +++ b/provider/gpkg/gpkg_register_internal_test.go @@ -1,3 +1,4 @@ +//go:build cgo // +build cgo package gpkg @@ -1095,7 +1096,7 @@ func TestCleanup(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := NewTileProvider(tc.config) + _, err := NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return diff --git a/provider/gpkg/gpkg_test.go b/provider/gpkg/gpkg_test.go index 8941e74af..b3142dd13 100644 --- a/provider/gpkg/gpkg_test.go +++ b/provider/gpkg/gpkg_test.go @@ -184,7 +184,7 @@ func TestNewTileProvider(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { if err.Error() != tc.expectedErr.Error() { t.Errorf("expectedErr %v got %v", tc.expectedErr, err) @@ -263,7 +263,7 @@ func TestTileFeatures(t *testing.T) { return func(t *testing.T) { t.Parallel() - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("new tile, expected nil got %v", err) return @@ -411,7 +411,7 @@ func TestConfigs(t *testing.T) { fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - p, err := gpkg.NewTileProvider(tc.config) + p, err := gpkg.NewTileProvider(tc.config, nil) if err != nil { t.Fatalf("err creating NewTileProvider: %v", err) return @@ -593,7 +593,7 @@ func TestOpenNonExistantFile(t *testing.T) { os.Remove(NONEXISTANTFILE) fn := func(tc tcase) func(*testing.T) { return func(t *testing.T) { - _, err := gpkg.NewTileProvider(tc.config) + _, err := gpkg.NewTileProvider(tc.config, nil) if reflect.TypeOf(err) != reflect.TypeOf(tc.err) { t.Errorf("expected error, expected %v got %v", tc.err, err) } @@ -601,13 +601,13 @@ func TestOpenNonExistantFile(t *testing.T) { } tests := map[string]tcase{ - "empty": tcase{ + "empty": { config: dict.Dict{ gpkg.ConfigKeyFilePath: "", }, err: gpkg.ErrInvalidFilePath{FilePath: ""}, }, - "nonexistance": tcase{ + "nonexistance": { // should not exists config: dict.Dict{ gpkg.ConfigKeyFilePath: NONEXISTANTFILE, diff --git a/provider/map.go b/provider/map.go new file mode 100644 index 000000000..d59ccd0b9 --- /dev/null +++ b/provider/map.go @@ -0,0 +1,14 @@ +package provider + +import "github.com/go-spatial/tegola/internal/env" + +// A Map represents a map in the Tegola Config file. +type Map struct { + Name env.String `toml:"name"` + Attribution env.String `toml:"attribution"` + Bounds []env.Float `toml:"bounds"` + Center [3]env.Float `toml:"center"` + Layers []MapLayer `toml:"layers"` + Parameters []QueryParameter `toml:"params"` + TileBuffer *env.Int `toml:"tile_buffer"` +} diff --git a/provider/map_layer.go b/provider/map_layer.go new file mode 100644 index 000000000..4a29d61a4 --- /dev/null +++ b/provider/map_layer.go @@ -0,0 +1,51 @@ +package provider + +import ( + "fmt" + "strings" + + "github.com/go-spatial/tegola/internal/env" +) + +// MapLayer represents a the config for a layer in a map +type MapLayer struct { + // Name is optional. If it's not defined the name of the ProviderLayer will be used. + // Name can also be used to group multiple ProviderLayers under the same namespace. + Name env.String `toml:"name"` + ProviderLayer env.String `toml:"provider_layer"` + MinZoom *env.Uint `toml:"min_zoom"` + MaxZoom *env.Uint `toml:"max_zoom"` + DefaultTags env.Dict `toml:"default_tags"` + // DontSimplify indicates whether feature simplification should be applied. + // We use a negative in the name so the default is to simplify + DontSimplify env.Bool `toml:"dont_simplify"` + // DontClip indicates whether feature clipping should be applied. + // We use a negative in the name so the default is to clipping + DontClip env.Bool `toml:"dont_clip"` + // DontClip indicates whether feature cleaning (e.g. make valid) should be applied. + // We use a negative in the name so the default is to clean + DontClean env.Bool `toml:"dont_clean"` +} + +// ProviderLayerName returns the names of the layer and provider or an error +func (ml MapLayer) ProviderLayerName() (provider, layer string, err error) { + // split the provider layer (syntax is provider.layer) + plParts := strings.Split(string(ml.ProviderLayer), ".") + if len(plParts) != 2 { + // TODO (beymak): Properly handle the error + return "", "", fmt.Errorf("config: invalid provider layer name (%v)", ml.ProviderLayer) + // return "", "", ErrInvalidProviderLayerName{ProviderLayerName: string(ml.ProviderLayer)} + } + return plParts[0], plParts[1], nil +} + +// GetName will return the user-defined Layer name from the config, +// or if the name is empty, return the name of the layer associated with +// the provider +func (ml MapLayer) GetName() (string, error) { + if ml.Name != "" { + return string(ml.Name), nil + } + _, name, err := ml.ProviderLayerName() + return name, err +} diff --git a/provider/mvt_provider.go b/provider/mvt_provider.go index 558682a9f..4f192fec8 100644 --- a/provider/mvt_provider.go +++ b/provider/mvt_provider.go @@ -10,8 +10,8 @@ type MVTTiler interface { Layerer // MVTForLayers will return a MVT byte array or an error for the given layer names. - MVTForLayers(ctx context.Context, tile Tile, params map[string]QueryParameter, layers []Layer) ([]byte, error) + MVTForLayers(ctx context.Context, tile Tile, params Params, layers []Layer) ([]byte, error) } // MVTInitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type MVTInitFunc func(dicter dict.Dicter) (MVTTiler, error) +type MVTInitFunc func(dicter dict.Dicter, maps []Map) (MVTTiler, error) diff --git a/provider/paramater_decoders.go b/provider/paramater_decoders.go new file mode 100644 index 000000000..d1a2f7b91 --- /dev/null +++ b/provider/paramater_decoders.go @@ -0,0 +1,19 @@ +package provider + +import "strconv" + +// ParamTypeDecoders is a collection of parsers for different types of user-defined parameters +var ParamTypeDecoders = map[string]func(string) (interface{}, error){ + "int": func(s string) (interface{}, error) { + return strconv.Atoi(s) + }, + "float": func(s string) (interface{}, error) { + return strconv.ParseFloat(s, 32) + }, + "string": func(s string) (interface{}, error) { + return s, nil + }, + "bool": func(s string) (interface{}, error) { + return strconv.ParseBool(s) + }, +} diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 23a907099..8c7be5014 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -171,6 +171,7 @@ const ( ) const ( + ConfigKeyName = "name" ConfigKeyURI = "uri" ConfigKeyHost = "host" ConfigKeyPort = "port" @@ -406,26 +407,24 @@ func BuildDBConfig(uri string) (*pgxpool.Config, error) { // trying to create a driver. This Provider supports the following fields // in the provided map[string]interface{} map: // -// host (string): [Required] postgis database host -// port (int): [Required] postgis database port (required) -// database (string): [Required] postgis database name -// user (string): [Required] postgis database user -// password (string): [Required] postgis database password -// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) -// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. -// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties +// name (string): [Required] name of the provider +// host (string): [Required] postgis database host +// port (int): [Required] postgis database port (required) +// database (string): [Required] postgis database name +// user (string): [Required] postgis database user +// password (string): [Required] postgis database password +// srid (int): [Optional] The default SRID for the provider. Defaults to WebMercator (3857) but also supports WGS84 (4326) +// max_connections : [Optional] The max connections to maintain in the connection pool. Default is 100. 0 means no max. +// layers (map[string]struct{}) — This is map of layers keyed by the layer name. supports the following properties // -// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. -// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. -// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom -// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid -// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. -// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). -// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: -// -// !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. -// !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. -func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { +// name (string): [Required] the name of the layer. This is used to reference this layer from map layers. +// tablename (string): [*Required] the name of the database table to query against. Required if sql is not defined. +// geometry_fieldname (string): [Optional] the name of the filed which contains the geometry for the feature. defaults to geom +// id_fieldname (string): [Optional] the name of the feature id field. defaults to gid +// fields ([]string): [Optional] a list of fields to include alongside the feature. Can be used if sql is not defined. +// srid (int): [Optional] the SRID of the layer. Supports 3857 (WebMercator) or 4326 (WGS84). +// sql (string): [*Required] custom SQL to use use. Required if tablename is not defined. Supports the following tokens: +func CreateProvider(config dict.Dicter, maps []provider.Map, providerType string) (*Provider, error) { uri, params, err := BuildURI(config) if err != nil { return nil, err @@ -453,7 +452,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) dbconfig, err := BuildDBConfig(uri.String()) if err != nil { - return nil, fmt.Errorf("failed while building db config: %v", err) + return nil, fmt.Errorf("failed while building db config: %w", err) } srid := DefaultSRID @@ -472,7 +471,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) pool, err := pgxpool.ConnectConfig(context.Background(), &p.config) if err != nil { - return nil, fmt.Errorf("failed while creating connection pool: %v", err) + return nil, fmt.Errorf("failed while creating connection pool: %w", err) } p.pool = &connectionPoolCollector{Pool: pool} @@ -488,7 +487,7 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) lName, err := layer.String(ConfigKeyLayerName, nil) if err != nil { - return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %v", i, err) + return nil, fmt.Errorf("for layer (%v) we got the following error trying to get the layer's name field: %w", i, err) } if j, ok := lyrsSeen[lName]; ok { @@ -605,7 +604,12 @@ func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w", l.name, err) } } else { - if err = p.inspectLayerGeomType(&l); err != nil { + pname, err := config.String(ConfigKeyName, nil) + if err != nil { + return nil, err + } + + if err = p.inspectLayerGeomType(pname, &l, maps); err != nil { return nil, fmt.Errorf("error fetching geometry type for layer (%v): %w\nif custom parameters are used, remember to set %s for the provider", l.name, err, ConfigKeyGeomType) } } @@ -698,7 +702,7 @@ func (p Provider) setLayerGeomType(l *Layer, geomType string) error { // inspectLayerGeomType sets the geomType field on the layer by running the SQL // and reading the geom type in the result set -func (p Provider) inspectLayerGeomType(l *Layer) error { +func (p Provider) inspectLayerGeomType(pname string, l *Layer, maps []provider.Map) error { var err error // we want to know the geom type instead of returning the geom data so we modify the SQL @@ -732,12 +736,20 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { return err } - // remove all parameter tokens for inspection - // crossing our fingers that the query is still valid 🤞 - // if not, the user will have to specify `geometry_type` in the config - sql = stripParams(sql) + // substitute default values to parameter + params := extractQueryParamValues(pname, maps, l) + + args := make([]interface{}, 0) + sql = params.ReplaceParams(sql, &args) + + if provider.ParameterTokenRegexp.MatchString(sql) { + // remove all parameter tokens for inspection + // crossing our fingers that the query is still valid 🤞 + // if not, the user will have to specify `geometry_type` in the config + sql = provider.ParameterTokenRegexp.ReplaceAllString(sql, "") + } - rows, err := p.pool.Query(context.Background(), sql) + rows, err := p.pool.Query(context.Background(), sql, args...) if err != nil { return err } @@ -804,8 +816,7 @@ func (p Provider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures adheres to the provider.Tiler interface -// TODO (bemyak): Make an actual use of QueryParams -func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider.Tile, params provider.Params, fn func(f *provider.Feature) error) error { var mapName string { @@ -828,7 +839,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. // replace configured query parameters if any args := make([]interface{}, 0) - sql = provider.ReplaceParams(queryParams, sql, &args) + sql = params.ReplaceParams(sql, &args) if err != nil { return err } @@ -941,7 +952,7 @@ func (p Provider) TileFeatures(ctx context.Context, layer string, tile provider. return rows.Err() } -func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params map[string]provider.QueryParameter, layers []provider.Layer) ([]byte, error) { +func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params provider.Params, layers []provider.Layer) ([]byte, error) { var ( err error sqls = make([]string, 0, len(layers)) @@ -977,7 +988,7 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, params m } // replace configured query parameters if any - sql = provider.ReplaceParams(params, sql, &args) + sql = params.ReplaceParams(sql, &args) // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(any_element row, text name, integer extent, text geom_name, text feature_id_name) diff --git a/provider/postgis/postgis_internal_test.go b/provider/postgis/postgis_internal_test.go index bd1346bd0..bec0787b1 100644 --- a/provider/postgis/postgis_internal_test.go +++ b/provider/postgis/postgis_internal_test.go @@ -98,7 +98,8 @@ func TestMVTProviders(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - prvd, err := NewMVTTileProvider(config) + config[ConfigKeyName] = "provider_name" + prvd, err := NewMVTTileProvider(config, nil) // for now we will just check the length of the bytes. if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { @@ -168,7 +169,8 @@ func TestLayerGeomType(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(DefaultEnvConfig) - provider, err := NewTileProvider(config) + config[ConfigKeyName] = "provider_name" + provider, err := NewTileProvider(config, nil) if tc.err != "" { if err == nil || !strings.Contains(err.Error(), tc.err) { t.Logf("error %#v", err) @@ -276,11 +278,11 @@ func TestLayerGeomType(t *testing.T) { TCConfig: TCConfig{ ConfigOverride: map[string]interface{}{ ConfigKeyURI: fmt.Sprintf("postgres://%v:%v@%v:%v/%v", - defaultEnvConfig["user"], - defaultEnvConfig["password"], - defaultEnvConfig["host"], - defaultEnvConfig["port"], - defaultEnvConfig["database"], + DefaultEnvConfig["user"], + DefaultEnvConfig["password"], + DefaultEnvConfig["host"], + DefaultEnvConfig["port"], + DefaultEnvConfig["database"], ), ConfigKeyHost: "", ConfigKeyPort: "", diff --git a/provider/postgis/postgis_test.go b/provider/postgis/postgis_test.go index 3907e1d90..22f4f3ffc 100644 --- a/provider/postgis/postgis_test.go +++ b/provider/postgis/postgis_test.go @@ -165,7 +165,8 @@ func TestNewTileProvider(t *testing.T) { fn := func(tc postgis.TCConfig) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - _, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + _, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unable to create a new provider. err: %v", err) return @@ -203,7 +204,8 @@ func TestTileFeatures(t *testing.T) { fn := func(tc tcase) func(t *testing.T) { return func(t *testing.T) { config := tc.Config(postgis.DefaultEnvConfig) - p, err := postgis.NewTileProvider(config) + config[postgis.ConfigKeyName] = "provider_name" + p, err := postgis.NewTileProvider(config, nil) if err != nil { t.Errorf("unexpected error; unable to create a new provider, expected: nil Got %v", err) return diff --git a/provider/postgis/register.go b/provider/postgis/register.go index ab0939be5..4c3a271b2 100644 --- a/provider/postgis/register.go +++ b/provider/postgis/register.go @@ -40,9 +40,9 @@ const ( // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { - return CreateProvider(config, ProviderType) +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { + return CreateProvider(config, maps, ProviderType) } -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { - return CreateProvider(config, MVTProviderType) +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { + return CreateProvider(config, maps, MVTProviderType) } diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 1d68c6b07..7dcb5df07 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -10,6 +10,7 @@ import ( "github.com/go-spatial/tegola" "github.com/go-spatial/tegola/basic" "github.com/go-spatial/tegola/config" + "github.com/go-spatial/tegola/internal/env" "github.com/go-spatial/tegola/internal/log" "github.com/go-spatial/tegola/provider" "github.com/jackc/pgproto3/v2" @@ -173,9 +174,25 @@ func replaceTokens(sql string, lyr *Layer, tile provider.Tile, withBuffer bool) return tokenReplacer.Replace(uppercaseTokenSQL), nil } -// stripParams will remove all parameter tokens from the query -func stripParams(sql string) string { - return provider.ParameterTokenRegexp.ReplaceAllString(sql, "") +// extractQueryParamValues finds default values for SQL tokens and constructs query parameter values out of them +func extractQueryParamValues(pname string, maps []provider.Map, layer *Layer) provider.Params { + result := make(provider.Params, 0) + + expectedMapName := fmt.Sprintf("%s.%s", pname, layer.name) + for _, m := range maps { + for _, l := range m.Layers { + if l.ProviderLayer == env.String(expectedMapName) { + for _, p := range m.Parameters { + pv, err := p.ToDefaultValue() + if err == nil { + result[p.Token] = pv + } + } + } + } + } + + return result } // uppercaseTokens converts all !tokens! to uppercase !TOKENS!. Tokens can diff --git a/provider/postgis/util_internal_test.go b/provider/postgis/util_internal_test.go index a662a047b..cf67d37b9 100644 --- a/provider/postgis/util_internal_test.go +++ b/provider/postgis/util_internal_test.go @@ -74,128 +74,6 @@ func TestReplaceTokens(t *testing.T) { } } -func TestReplaceParams(t *testing.T) { - type tcase struct { - params map[string]provider.QueryParameter - sql string - expectedSql string - expectedArgs []interface{} - } - - fn := func(tc tcase) func(t *testing.T) { - return func(t *testing.T) { - args := make([]interface{}, 0) - out := provider.ReplaceParams(tc.params, tc.sql, &args) - - if out != tc.expectedSql { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) - return - } - - if len(tc.expectedArgs) != len(args) { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - for i, arg := range tc.expectedArgs { - if arg != args[i] { - t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) - return - } - } - } - } - - tests := map[string]tcase{ - "nil params": { - params: nil, - sql: "SELECT * FROM table", - expectedSql: "SELECT * FROM table", - expectedArgs: []interface{}{}, - }, - "int replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: 1, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{1}, - }, - "string replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: "test", - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{"test"}, - }, - "null replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "?", - Value: nil, - }, - }, - sql: "SELECT * FROM table WHERE PARAM = !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM = $1", - expectedArgs: []interface{}{nil}, - }, - "complex sql replacement": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "WHERE PARAM=?", - Value: 1, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table WHERE PARAM=$1", - expectedArgs: []interface{}{1}, - }, - "subquery removal": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "", - Value: nil, - }, - }, - sql: "SELECT * FROM table !PARAM!", - expectedSql: "SELECT * FROM table ", - expectedArgs: []interface{}{}, - }, - "multiple params": { - params: map[string]provider.QueryParameter{ - "!PARAM!": { - Token: "!PARAM!", - SQL: "???", - Value: 1, - }, - "!PARAM2!": { - Token: "!PARAM2!", - SQL: "???", - Value: 2, - }, - }, - sql: "!PARAM!!PARAM2!", - expectedSql: "$1$1$1$2$2$2", - expectedArgs: []interface{}{1, 2}, - }, - } - - for name, tc := range tests { - t.Run(name, fn(tc)) - } -} - func TestUppercaseTokens(t *testing.T) { type tcase struct { str string diff --git a/provider/provider.go b/provider/provider.go index 6fc2fe2c2..6813892cb 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "regexp" - "strings" "github.com/go-spatial/geom" "github.com/go-spatial/geom/slippy" @@ -109,59 +108,13 @@ type Tile interface { // ParameterTokenRegexp to validate QueryParameters var ParameterTokenRegexp = regexp.MustCompile("![a-zA-Z0-9_-]+!") -// Query parameter holds normalized parameter data ready to be inserted in the -// final query -type QueryParameter struct { - // Token to replace e.g., !TOKEN! - Token string - // SQL expression to be inserted. Contains "?" that will be replaced with an - // ordinal argument e.g., "$1" - SQL string - // Value that will be passed to the final query in arguments list - Value interface{} - // Raw parameter values for debugging and monitoring - RawValues map[string]string -} - -// ReplaceParams substitutes configured query parameter tokens for their values -// within the provided SQL string -func ReplaceParams(params map[string]QueryParameter, sql string, args *[]interface{}) string { - if params == nil { - return sql - } - - for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { - param := params[token] - - // Replace every ? in the param's SQL with a positional argument - paramSQL := "" - argFound := false - for _, c := range param.SQL { - if c == '?' { - if !argFound { - *args = append(*args, param.Value) - argFound = true - } - paramSQL += fmt.Sprintf("$%d", len(*args)) - } else { - paramSQL += string(c) - } - } - - // Finally, replace current token with the prepared SQL - sql = strings.Replace(sql, token, paramSQL, 1) - } - - return sql -} - // Tiler is a Layers that allows one to encode features in that layer type Tiler interface { Layerer // TileFeature will stream decoded features to the callback function fn // if fn returns ErrCanceled, the TileFeatures method should stop processing - TileFeatures(ctx context.Context, layer string, t Tile, queryParams map[string]QueryParameter, fn func(f *Feature) error) error + TileFeatures(ctx context.Context, layer string, t Tile, params Params, fn func(f *Feature) error) error } // TilerUnion represents either a Std Tiler or and MVTTiler; only one should be not nil. @@ -183,7 +136,7 @@ func (tu TilerUnion) Layers() ([]LayerInfo, error) { } // InitFunc initialize a provider given a config map. The init function should validate the config map, and report any errors. This is called by the For function. -type InitFunc func(dicter dict.Dicter) (Tiler, error) +type InitFunc func(dicter dict.Dicter, maps []Map) (Tiler, error) // CleanupFunc is called to when the system is shutting down, this allows the provider to cleanup. type CleanupFunc func() @@ -280,7 +233,7 @@ func Drivers(types ...providerType) (l []string) { // For function returns a configure provider of the given type; The provider may be a mvt provider or // a std provider. The correct entry in TilerUnion will not be nil. If there is an error both entries // will be nil. -func For(name string, config dict.Dicter) (val TilerUnion, err error) { +func For(name string, config dict.Dicter, maps []Map) (val TilerUnion, err error) { var ( driversList = Drivers() ) @@ -292,11 +245,11 @@ func For(name string, config dict.Dicter) (val TilerUnion, err error) { return val, ErrUnknownProvider{KnownProviders: driversList, Name: name} } if p.init != nil { - val.Std, err = p.init(config) + val.Std, err = p.init(config, maps) return val, err } if p.mvtInit != nil { - val.Mvt, err = p.mvtInit(config) + val.Mvt, err = p.mvtInit(config, maps) return val, err } return val, ErrInvalidRegisteredProvider{Name: name} diff --git a/provider/provider_test.go b/provider/provider_test.go index 220ef94a4..d67a98a93 100644 --- a/provider/provider_test.go +++ b/provider/provider_test.go @@ -12,7 +12,7 @@ func TestProviderInterface(t *testing.T) { stdName = provider.TypeStd.Prefix() + test.Name mvtName = provider.TypeMvt.Prefix() + test.Name ) - if _, err := provider.For(stdName, nil); err != nil { + if _, err := provider.For(stdName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } @@ -23,7 +23,7 @@ func TestProviderInterface(t *testing.T) { if test.Count != 0 { t.Errorf(" expected count , expected 0 got %v", test.Count) } - if _, err := provider.For(mvtName, nil); err != nil { + if _, err := provider.For(mvtName, nil, nil); err != nil { t.Errorf("retrieve provider err , expected nil got %v", err) return } diff --git a/provider/query_parameter.go b/provider/query_parameter.go new file mode 100644 index 000000000..181169bcb --- /dev/null +++ b/provider/query_parameter.go @@ -0,0 +1,65 @@ +package provider + +import ( + "fmt" + "strings" +) + +// QueryParameter represents an HTTP query parameter specified for use with +// a given map instance. +type QueryParameter struct { + Name string `toml:"name"` + Token string `toml:"token"` + Type string `toml:"type"` + SQL string `toml:"sql"` + // DefaultSQL replaces SQL if param wasn't passed. Either default_sql or + // default_value can be specified + DefaultSQL string `toml:"default_sql"` + DefaultValue string `toml:"default_value"` +} + +// Normalize normalizes param and sets default values +func (param *QueryParameter) Normalize() { + param.Token = strings.ToUpper(param.Token) + + if len(param.SQL) == 0 { + param.SQL = "?" + } +} + +func (param *QueryParameter) ToValue(rawValue string) (QueryParameterValue, error) { + val, err := ParamTypeDecoders[param.Type](rawValue) + if err != nil { + return QueryParameterValue{}, err + } + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: rawValue, + }, nil +} + +func (param *QueryParameter) ToDefaultValue() (QueryParameterValue, error) { + if len(param.DefaultValue) > 0 { + val, err := ParamTypeDecoders[param.Type](param.DefaultValue) + return QueryParameterValue{ + Token: param.Token, + SQL: param.SQL, + Value: val, + RawParam: param.Name, + RawValue: "", + }, err + } + if len(param.DefaultSQL) > 0 { + return QueryParameterValue{ + Token: param.Token, + SQL: param.DefaultSQL, + Value: nil, + RawParam: param.Name, + RawValue: "", + }, nil + } + return QueryParameterValue{}, fmt.Errorf("the required parameter %s is not specified", param.Name) +} diff --git a/provider/query_parameter_value.go b/provider/query_parameter_value.go new file mode 100644 index 000000000..10fec3170 --- /dev/null +++ b/provider/query_parameter_value.go @@ -0,0 +1,76 @@ +package provider + +import ( + "fmt" + "strings" +) + +// Query parameter holds normalized parameter data ready to be inserted in the +// final query +type QueryParameterValue struct { + // Token to replace e.g., !TOKEN! + Token string + // SQL expression to be inserted. Contains "?" that will be replaced with an + // ordinal argument e.g., "$1" + SQL string + // Value that will be passed to the final query in arguments list + Value interface{} + // Raw parameter and value for debugging and monitoring + RawParam string + // RawValue will be "" if the param wasn't passed and defaults were used + RawValue string +} + +type Params map[string]QueryParameterValue + +// ReplaceParams substitutes configured query parameter tokens for their values +// within the provided SQL string +func (params Params) ReplaceParams(sql string, args *[]interface{}) string { + if params == nil { + return sql + } + + var ( + cache = make(map[string]string) + sb strings.Builder + ) + + for _, token := range ParameterTokenRegexp.FindAllString(sql, -1) { + resultSQL, ok := cache[token] + if ok { + // Already have it cached, replace the token and move on. + sql = strings.ReplaceAll(sql, token, resultSQL) + continue + } + + param, ok := params[token] + if !ok { + // Unknown token, ignoring + continue + } + + sb.Reset() + sb.Grow(len(param.SQL)) + argFound := false + + // Replace every `?` in the param's SQL with a positional argument + for _, c := range param.SQL { + if c != '?' { + sb.WriteRune(c) + continue + } + + if !argFound { + *args = append(*args, param.Value) + argFound = true + } + sb.WriteString(fmt.Sprintf("$%d", len(*args))) + } + + resultSQL = sb.String() + cache[token] = resultSQL + sql = strings.ReplaceAll(sql, token, resultSQL) + } + + return sql +} diff --git a/provider/query_parameter_value_test.go b/provider/query_parameter_value_test.go new file mode 100644 index 000000000..204bca97f --- /dev/null +++ b/provider/query_parameter_value_test.go @@ -0,0 +1,137 @@ +package provider + +import "testing" + +func TestReplaceParams(t *testing.T) { + type tcase struct { + params Params + sql string + expectedSql string + expectedArgs []interface{} + } + + fn := func(tc tcase) func(t *testing.T) { + return func(t *testing.T) { + args := make([]interface{}, 0) + out := tc.params.ReplaceParams(tc.sql, &args) + + if out != tc.expectedSql { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedSql, out) + return + } + + if len(tc.expectedArgs) != len(args) { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + for i, arg := range tc.expectedArgs { + if arg != args[i] { + t.Errorf("expected \n \t%v\n out \n \t%v", tc.expectedArgs, args) + return + } + } + } + } + + tests := map[string]tcase{ + "nil params": { + params: nil, + sql: "SELECT * FROM table", + expectedSql: "SELECT * FROM table", + expectedArgs: []interface{}{}, + }, + "int replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{1}, + }, + "string replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: "test", + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{"test"}, + }, + "null replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: nil, + }, + }, + sql: "SELECT * FROM table WHERE param = !PARAM!", + expectedSql: "SELECT * FROM table WHERE param = $1", + expectedArgs: []interface{}{nil}, + }, + "complex sql replacement": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "WHERE param=?", + Value: 1, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table WHERE param=$1", + expectedArgs: []interface{}{1}, + }, + "subquery removal": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "", + Value: nil, + }, + }, + sql: "SELECT * FROM table !PARAM!", + expectedSql: "SELECT * FROM table ", + expectedArgs: []interface{}{}, + }, + "multiple params": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "? ? ?", + Value: 1, + }, + "!PARAM2!": { + Token: "!PARAM2!", + SQL: "???", + Value: 2, + }, + }, + sql: "!PARAM! !PARAM2! !PARAM!", + expectedSql: "$1 $1 $1 $2$2$2 $1 $1 $1", + expectedArgs: []interface{}{1, 2}, + }, + "unknown token": { + params: Params{ + "!PARAM!": { + Token: "!PARAM!", + SQL: "?", + Value: 1, + }, + }, + sql: "!NOT_PARAM! !PARAM! !NOT_PARAM!", + expectedSql: "!NOT_PARAM! $1 !NOT_PARAM!", + expectedArgs: []interface{}{1}, + }, + } + + for name, tc := range tests { + t.Run(name, fn(tc)) + } +} diff --git a/provider/test/emptycollection/provider.go b/provider/test/emptycollection/provider.go index 458a4e540..eef8d9ac0 100644 --- a/provider/test/emptycollection/provider.go +++ b/provider/test/emptycollection/provider.go @@ -19,7 +19,7 @@ func init() { } // NewProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { Count++ return &TileProvider{}, nil } @@ -40,7 +40,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TilFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box _, srid := t.Extent() diff --git a/provider/test/provider.go b/provider/test/provider.go index ca0942693..011242afb 100644 --- a/provider/test/provider.go +++ b/provider/test/provider.go @@ -28,7 +28,7 @@ func init() { } // NewTileProvider setups a test provider. there are not currently any config params supported -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { +func NewTileProvider(config dict.Dicter, maps []provider.Map) (provider.Tiler, error) { lock.Lock() Count++ lock.Unlock() @@ -37,7 +37,7 @@ func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { // NewMVTTileProvider setups a test provider for mvt tiles providers. The only supported parameter is // "test_file", which should point to a mvt tile file to return for MVTForLayers -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { +func NewMVTTileProvider(config dict.Dicter, maps []provider.Map) (provider.MVTTiler, error) { lock.Lock() MVTCount++ lock.Unlock() @@ -86,7 +86,7 @@ func (tp *TileProvider) Layers() ([]provider.LayerInfo, error) { } // TileFeatures always returns a feature with a polygon outlining the tile's Extent (not Buffered Extent) -func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams map[string]provider.QueryParameter, fn func(f *provider.Feature) error) error { +func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provider.Tile, queryParams provider.Params, fn func(f *provider.Feature) error) error { // get tile bounding box ext, srid := t.Extent() @@ -103,7 +103,7 @@ func (tp *TileProvider) TileFeatures(ctx context.Context, layer string, t provid } // MVTForLayers mocks out MVTForLayers by just returning the MVTTile bytes, this will never error -func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ map[string]provider.QueryParameter, _ []provider.Layer) ([]byte, error) { +func (tp *TileProvider) MVTForLayers(ctx context.Context, _ provider.Tile, _ provider.Params, _ []provider.Layer) ([]byte, error) { // TODO(gdey): fill this out. if tp == nil { return nil, nil diff --git a/server/handle_map_layer_zxy.go b/server/handle_map_layer_zxy.go index 9cfbfa65e..39345a3b0 100644 --- a/server/handle_map_layer_zxy.go +++ b/server/handle_map_layer_zxy.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/go-spatial/tegola/config" "github.com/go-spatial/tegola/observability" "github.com/go-spatial/tegola/provider" @@ -101,9 +100,11 @@ func (req *HandleMapLayerZXY) parseURI(r *http.Request) error { // map_name - map name in the config file // layer_name - name of the single map layer to render // z, x, y - tile coordinates as described in the Slippy Map Tilenames specification -// z - zoom level -// x - row -// y - column +// +// z - zoom level +// x - row +// y - column +// // param - configurable query parameters and their values func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { // parse our URI @@ -162,7 +163,7 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // check for query parameters and populate param map with their values - params, err := req.extractParameters(r) + params, err := extractParameters(m, r) if err != nil { log.Error(err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -206,44 +207,28 @@ func (req HandleMapLayerZXY) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (req *HandleMapLayerZXY) extractParameters(r *http.Request) (map[string]provider.QueryParameter, error) { - var params map[string]provider.QueryParameter - if req.Atlas.HasParams(req.mapName) { - params = make(map[string]provider.QueryParameter) +func extractParameters(m atlas.Map, r *http.Request) (provider.Params, error) { + var params provider.Params + if m.Params != nil && len(m.Params) > 0 { + params = make(provider.Params) err := r.ParseForm() if err != nil { return nil, err } - for _, param := range req.Atlas.GetParams(req.mapName) { + for _, param := range m.Params { if r.Form.Has(param.Name) { - val, err := config.ParamTypeDecoders[param.Type](r.Form.Get(param.Name)) + val, err := param.ToValue(r.Form.Get(param.Name)) if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultValue) > 0 { - val, err := config.ParamTypeDecoders[param.Type](param.DefaultValue) + params[param.Token] = val + } else { + p, err := param.ToDefaultValue() if err != nil { return nil, err } - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.SQL, - Value: val, - } - } else if len(param.DefaultSQL) > 0 { - params[param.Token] = provider.QueryParameter{ - Token: param.Type, - SQL: param.DefaultSQL, - Value: nil, - } - } else { - return nil, fmt.Errorf("the required parameter %s is not specified", param.Name) + params[param.Token] = p } } } diff --git a/tile.go b/tile.go index 609238f6f..38ecd7bc5 100644 --- a/tile.go +++ b/tile.go @@ -17,7 +17,7 @@ const ( var UnknownConversionError = fmt.Errorf("do not know how to convert value to requested value") -//Tile slippy map tilenames +// Tile slippy map tilenames // http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames type Tile struct { Z uint @@ -198,7 +198,7 @@ func (t *Tile) ZLevel() uint { return t.Z } -//ZRes takes a web mercator zoom level and returns the pixel resolution for that +// ZRes takes a web mercator zoom level and returns the pixel resolution for that // scale, assuming t.Extent x t.Extent pixel tiles. Non-integer zoom levels are accepted. // ported from: https://raw.githubusercontent.com/mapbox/postgis-vt-util/master/postgis-vt-util.sql // 40075016.6855785 is the equator in meters for WGS84 at z=0