From 583da97260155af004cfaaea9fe8b44e3d749217 Mon Sep 17 00:00:00 2001 From: jgiannelos Date: Wed, 6 Jan 2021 17:00:53 +0100 Subject: [PATCH] Fix SQL parsing for MVT provider * Make ID field optional * Fix geometry sniffing for MVT queries * Fix handling MVT provider SQL when using (SELECT ...) as alias queries --- provider/postgis/postgis.go | 24 ++++++++++++++++++++---- provider/postgis/register.go | 8 ++++++-- provider/postgis/util.go | 28 ++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/provider/postgis/postgis.go b/provider/postgis/postgis.go index 6691a7c4e..785051547 100644 --- a/provider/postgis/postgis.go +++ b/provider/postgis/postgis.go @@ -35,6 +35,7 @@ type Provider struct { const ( // We quote the field and table names to prevent colliding with postgres keywords. stdSQL = `SELECT %[1]v FROM %[2]v WHERE "%[3]v" && ` + bboxToken + mvtSQL = `SELECT %[1]v FROM %[2]v` // SQL to get the column names, without hitting the information_schema. Though it might be better to hit the information_schema. fldsSQL = `SELECT * FROM %[1]v LIMIT 0;` @@ -100,7 +101,7 @@ var isSelectQuery = regexp.MustCompile(`(?i)^((\s*)(--.*\n)?)*select`) // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func CreateProvider(config dict.Dicter) (*Provider, error) { +func CreateProvider(config dict.Dicter, providerType string) (*Provider, error) { host, err := config.String(ConfigKeyHost, nil) if err != nil { @@ -297,7 +298,7 @@ func CreateProvider(config dict.Dicter) (*Provider, error) { // Tablename and Fields will be used to build the query. // We need to do some work. We need to check to see Fields contains the geom and gid fields // and if not add them to the list. If Fields list is empty/nil we will use '*' for the field list. - l.sql, err = genSQL(&l, p.pool, tblName, fields, true) + l.sql, err = genSQL(&l, p.pool, tblName, fields, true, providerType) if err != nil { return nil, fmt.Errorf("could not generate sql, for layer(%v): %v", lname, err) } @@ -420,9 +421,15 @@ func (p Provider) inspectLayerGeomType(l *Layer) error { // https://github.com/go-spatial/tegola/issues/180 // // case insensitive search + re := regexp.MustCompile(`(?i)ST_AsBinary`) sql := re.ReplaceAllString(l.sql, "ST_GeometryType") + re = regexp.MustCompile(`(?i)(ST_AsMVTGeom\(.*\))`) + if re.MatchString(sql) { + sql = fmt.Sprintf("SELECT ST_GeometryType(%v) FROM (%v) as q", l.geomField, sql) + } + // we only need a single result set to sniff out the geometry type sql = fmt.Sprintf("%v LIMIT 1", sql) @@ -638,12 +645,21 @@ func (p Provider) MVTForLayers(ctx context.Context, tile provider.Tile, layers [ // ref: https://postgis.net/docs/ST_AsMVT.html // bytea ST_AsMVT(anyelement row, text name, integer extent, text geom_name, text feature_id_name) + + var featureIDName string + + if l.IDFieldName() == "" { + featureIDName = "NULL" + } else { + featureIDName = fmt.Sprintf(`'%s'`, l.IDFieldName()) + } + sqls = append(sqls, fmt.Sprintf( - `(SELECT ST_AsMVT(q,'%s',%d,'%s','%s') AS data FROM (%s) AS q)`, + `(SELECT ST_AsMVT(q,'%s',%d,'%s',%s) AS data FROM (%s) AS q)`, layers[i].MVTName, tegola.DefaultExtent, l.GeomFieldName(), - l.IDFieldName(), + featureIDName, sql, )) } diff --git a/provider/postgis/register.go b/provider/postgis/register.go index f0dcfe287..ee5776e2d 100644 --- a/provider/postgis/register.go +++ b/provider/postgis/register.go @@ -35,5 +35,9 @@ func init() { // !BBOX! - [Required] will be replaced with the bounding box of the tile before the query is sent to the database. // !ZOOM! - [Optional] will be replaced with the "Z" (zoom) value of the requested tile. // -func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { return CreateProvider(config) } -func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { return CreateProvider(config) } +func NewTileProvider(config dict.Dicter) (provider.Tiler, error) { + return CreateProvider(config, "postgis") +} +func NewMVTTileProvider(config dict.Dicter) (provider.MVTTiler, error) { + return CreateProvider(config, "mvt_postgis") +} diff --git a/provider/postgis/util.go b/provider/postgis/util.go index 0ac87e1b7..a0cae3a17 100644 --- a/provider/postgis/util.go +++ b/provider/postgis/util.go @@ -16,8 +16,13 @@ import ( "github.com/jackc/pgx/pgtype" ) +// isMVT will return true if the provider is MVT based +func isMVT(providerType string) bool { + return providerType == "mvt_postgis" +} + // genSQL will fill in the SQL field of a layer given a pool, and list of fields. -func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string, buffer bool) (sql string, err error) { +func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string, buffer bool, providerType string) (sql string, err error) { // we need to hit the database to see what the fields are. if len(flds) == 0 { @@ -61,10 +66,19 @@ func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string, buffer // to avoid field names possibly colliding with Postgres keywords, // we wrap the field names in quotes + if fgeom == -1 { - flds = append(flds, fmt.Sprintf(`ST_AsBinary("%v") AS "%[1]v"`, l.geomField)) + if isMVT(providerType) { + flds = append(flds, fmt.Sprintf(`"%v" AS "%[1]v"`, l.geomField)) + } else { + flds = append(flds, fmt.Sprintf(`ST_AsBinary("%v") AS "%[1]v"`, l.geomField)) + } } else { - flds[fgeom] = fmt.Sprintf(`ST_AsBinary("%v") AS "%[1]v"`, l.geomField) + if isMVT(providerType) { + flds[fgeom] = fmt.Sprintf(`"%v" AS "%[1]v"`, l.geomField) + } else { + flds[fgeom] = fmt.Sprintf(`ST_AsBinary("%v") AS "%[1]v"`, l.geomField) + } } // add required id field @@ -74,7 +88,13 @@ func genSQL(l *Layer, pool *pgx.ConnPool, tblname string, flds []string, buffer selectClause := strings.Join(flds, ", ") - return fmt.Sprintf(stdSQL, selectClause, tblname, l.geomField), nil + sqlTmpl := stdSQL + + if isMVT(providerType) { + sqlTmpl = mvtSQL + } + + return fmt.Sprintf(sqlTmpl, selectClause, tblname, l.geomField), nil } const (