Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

When generating the seed list, we need the grid for the bounds #1024

Merged
merged 2 commits into from
Dec 31, 2024
Merged

When generating the seed list, we need the grid for the bounds #1024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eda30a2
Upated dependences
gdey Dec 31, 2024
141b759
When generating the seed list, we need the grid for the bounds
gdey Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ Tegola is a vector tile server delivering [Mapbox Vector Tiles](https://github.c

```
tegola is a vector tile server
Version: v0.17.0
Version: v0.21.0

Usage:
tegola [command]
Expand Down
36 changes: 27 additions & 9 deletions cmd/tegola/cmd/cache/seed_purge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ package cache

import (
"context"
"errors"
"fmt"
"runtime"
"strings"

"github.com/go-spatial/cobra"
"github.com/go-spatial/geom"
"github.com/go-spatial/geom/slippy"
"github.com/go-spatial/proj"
"github.com/go-spatial/tegola/atlas"
"github.com/go-spatial/tegola/internal/build"
gdcmd "github.com/go-spatial/tegola/internal/cmd"
Expand Down Expand Up @@ -41,15 +43,17 @@ Use "{{.CommandPath}} [command] --help" for more information about a command.{{e

// flag parameters
var (
// the amount of concurrency to use. defaults to the number of CPUs on the machine
// cacheConcurrency is the amount of concurrency to use. defaults to the number of CPUs on the machine
cacheConcurrency int
// cache overwrite
// cacheOverwrite determines if we should overwrite already existing files or skip them
cacheOverwrite bool
// bounds to cache within. default -180, -85.0511, 180, 85.0511
// cacheBounds is the bounds that the cache is within. defaults to -180, -85.0511, 180, 85.0511
cacheBounds string
// name of the map
// cacheBoundsSRID is the srid of grid system that the bounds is at. Should Default to 4326
cacheBoundsSRID int
// cacheMap is the name of the map
cacheMap string
// while seeding, on log output for tiles that take longer than this (in milliseconds) to render
// cacheLogThreshold is cache threshold while seeding, to log output for tiles that take longer than this (in milliseconds) to render
cacheLogThreshold int64
)

Expand All @@ -76,6 +80,7 @@ func init() {
SeedPurgeCmd.PersistentFlags().Int64VarP(&cacheLogThreshold, "log-threshold", "", 0, "during seeding, only log tiles that take this number of milliseconds or longer to render (default all tiles)")

SeedPurgeCmd.Flags().StringVarP(&cacheBounds, "bounds", "", "-180,-85.0511,180,85.0511", "lng/lat bounds to seed the cache with in the format: minx, miny, maxx, maxy")
SeedPurgeCmd.Flags().IntVarP(&cacheBoundsSRID, "bounds-srid", "", int(proj.EPSG4326), "the srid of the grid system for bounds.")

SeedPurgeCmd.PersistentPreRunE = seedPurgeCmdValidatePersistent
SeedPurgeCmd.PreRunE = seedPurgeCmdValidate
Expand Down Expand Up @@ -143,6 +148,15 @@ func seedPurgeCmdValidatePersistent(cmd *cobra.Command, args []string) error {
}

func seedPurgeCmdValidate(cmd *cobra.Command, args []string) (err error) {
// validate the cache-bounds-srid
if !proj.IsKnownConversionSRID(proj.EPSGCode(cacheBoundsSRID)) {
var str strings.Builder
str.WriteString(fmt.Sprintf("SRID=%d is not a know conversion ePSG code\n known codes are:", cacheBoundsSRID))
for _, code := range proj.AvailableConversions() {
str.WriteString(fmt.Sprintf(" %d\n", int(code)))
}
return errors.New(str.String())
}

// validate and set bounds flag
boundsParts := strings.Split(strings.TrimSpace(cacheBounds), ",")
Expand Down Expand Up @@ -191,27 +205,31 @@ func seedPurgeCommand(_ *cobra.Command, _ []string) (err error) {
}
}()

grid := slippy.NewGrid(proj.EPSGCode(cacheBoundsSRID), 0)

log.Info("zoom list: ", zooms)
tileChannel := generateTilesForBounds(ctx, seedPurgeBounds, zooms)
tileChannel := generateTilesForBounds(ctx, seedPurgeBounds, zooms, grid)

return doWork(ctx, tileChannel, seedPurgeMaps, cacheConcurrency, seedPurgeWorker)
}

func generateTilesForBounds(ctx context.Context, bounds [4]float64, zooms []uint) *TileChannel {
func generateTilesForBounds(ctx context.Context, bounds [4]float64, zooms []uint, grid slippy.TileGridder) *TileChannel {

tce := &TileChannel{
channel: make(chan slippy.Tile),
}

webmercatorGrid := slippy.NewGrid(3857, 0)
if grid == nil {
grid = slippy.NewGrid(proj.EPSGCode(cacheBoundsSRID), 0)
}

go func() {
defer tce.Close()

var extent geom.Extent = bounds
for _, z := range zooms {

tiles, err := slippy.FromBounds(webmercatorGrid, &extent, slippy.Zoom(z))
tiles, err := slippy.FromBounds(grid, &extent, slippy.Zoom(z))
if err != nil {
tce.setError(fmt.Errorf("got error trying to get tiles: %w", err))
tce.Close()
Expand Down
26 changes: 25 additions & 1 deletion cmd/tegola/cmd/cache/seed_purge_generator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing"

"github.com/go-spatial/geom/slippy"
"github.com/go-spatial/proj"
)

type sTiles []slippy.Tile
Expand Down Expand Up @@ -70,14 +71,15 @@ func TestGenerateTilesForBounds(t *testing.T) {
zooms []uint
bounds [4]float64
tiles sTiles
grid slippy.TileGridder
err error
}

fn := func(tc tcase) func(t *testing.T) {
return func(t *testing.T) {

// Setup up the generator.
tilechannel := generateTilesForBounds(context.Background(), tc.bounds, tc.zooms)
tilechannel := generateTilesForBounds(context.Background(), tc.bounds, tc.zooms, tc.grid)
tiles := make(sTiles, 0, len(tc.tiles))
for tile := range tilechannel.Channel() {
tiles = append(tiles, tile)
Expand Down Expand Up @@ -127,6 +129,28 @@ func TestGenerateTilesForBounds(t *testing.T) {
slippy.Tile{Z: 1, X: 1, Y: 1},
},
},
"min_zoom=1 max_zoom=1 bounds=5.9,45.8,10.5,47.8 WSG84": {
// see: https://github.com/go-spatial/tegola/issues/880#issuecomment-2556563251
zooms: []uint{10},
bounds: [4]float64{5.9, 45.8, 10.5, 47.8},
grid: slippy.NewGrid(proj.EPSG4326, 0),
tiles: sTiles{
slippy.Tile{Z: 10, X: 528, Y: 356}, slippy.Tile{Z: 10, X: 528, Y: 357}, slippy.Tile{Z: 10, X: 528, Y: 358}, slippy.Tile{Z: 10, X: 528, Y: 359}, slippy.Tile{Z: 10, X: 528, Y: 360}, slippy.Tile{Z: 10, X: 528, Y: 361}, slippy.Tile{Z: 10, X: 528, Y: 362}, slippy.Tile{Z: 10, X: 528, Y: 363}, slippy.Tile{Z: 10, X: 528, Y: 364}, slippy.Tile{Z: 10, X: 528, Y: 365},
slippy.Tile{Z: 10, X: 529, Y: 356}, slippy.Tile{Z: 10, X: 529, Y: 357}, slippy.Tile{Z: 10, X: 529, Y: 358}, slippy.Tile{Z: 10, X: 529, Y: 359}, slippy.Tile{Z: 10, X: 529, Y: 360}, slippy.Tile{Z: 10, X: 529, Y: 361}, slippy.Tile{Z: 10, X: 529, Y: 362}, slippy.Tile{Z: 10, X: 529, Y: 363}, slippy.Tile{Z: 10, X: 529, Y: 364}, slippy.Tile{Z: 10, X: 529, Y: 365},
slippy.Tile{Z: 10, X: 530, Y: 356}, slippy.Tile{Z: 10, X: 530, Y: 357}, slippy.Tile{Z: 10, X: 530, Y: 358}, slippy.Tile{Z: 10, X: 530, Y: 359}, slippy.Tile{Z: 10, X: 530, Y: 360}, slippy.Tile{Z: 10, X: 530, Y: 361}, slippy.Tile{Z: 10, X: 530, Y: 362}, slippy.Tile{Z: 10, X: 530, Y: 363}, slippy.Tile{Z: 10, X: 530, Y: 364}, slippy.Tile{Z: 10, X: 530, Y: 365},
slippy.Tile{Z: 10, X: 531, Y: 356}, slippy.Tile{Z: 10, X: 531, Y: 357}, slippy.Tile{Z: 10, X: 531, Y: 358}, slippy.Tile{Z: 10, X: 531, Y: 359}, slippy.Tile{Z: 10, X: 531, Y: 360}, slippy.Tile{Z: 10, X: 531, Y: 361}, slippy.Tile{Z: 10, X: 531, Y: 362}, slippy.Tile{Z: 10, X: 531, Y: 363}, slippy.Tile{Z: 10, X: 531, Y: 364}, slippy.Tile{Z: 10, X: 531, Y: 365},
slippy.Tile{Z: 10, X: 532, Y: 356}, slippy.Tile{Z: 10, X: 532, Y: 357}, slippy.Tile{Z: 10, X: 532, Y: 358}, slippy.Tile{Z: 10, X: 532, Y: 359}, slippy.Tile{Z: 10, X: 532, Y: 360}, slippy.Tile{Z: 10, X: 532, Y: 361}, slippy.Tile{Z: 10, X: 532, Y: 362}, slippy.Tile{Z: 10, X: 532, Y: 363}, slippy.Tile{Z: 10, X: 532, Y: 364}, slippy.Tile{Z: 10, X: 532, Y: 365},
slippy.Tile{Z: 10, X: 533, Y: 356}, slippy.Tile{Z: 10, X: 533, Y: 357}, slippy.Tile{Z: 10, X: 533, Y: 358}, slippy.Tile{Z: 10, X: 533, Y: 359}, slippy.Tile{Z: 10, X: 533, Y: 360}, slippy.Tile{Z: 10, X: 533, Y: 361}, slippy.Tile{Z: 10, X: 533, Y: 362}, slippy.Tile{Z: 10, X: 533, Y: 363}, slippy.Tile{Z: 10, X: 533, Y: 364}, slippy.Tile{Z: 10, X: 533, Y: 365},
slippy.Tile{Z: 10, X: 534, Y: 356}, slippy.Tile{Z: 10, X: 534, Y: 357}, slippy.Tile{Z: 10, X: 534, Y: 358}, slippy.Tile{Z: 10, X: 534, Y: 359}, slippy.Tile{Z: 10, X: 534, Y: 360}, slippy.Tile{Z: 10, X: 534, Y: 361}, slippy.Tile{Z: 10, X: 534, Y: 362}, slippy.Tile{Z: 10, X: 534, Y: 363}, slippy.Tile{Z: 10, X: 534, Y: 364}, slippy.Tile{Z: 10, X: 534, Y: 365},
slippy.Tile{Z: 10, X: 535, Y: 356}, slippy.Tile{Z: 10, X: 535, Y: 357}, slippy.Tile{Z: 10, X: 535, Y: 358}, slippy.Tile{Z: 10, X: 535, Y: 359}, slippy.Tile{Z: 10, X: 535, Y: 360}, slippy.Tile{Z: 10, X: 535, Y: 361}, slippy.Tile{Z: 10, X: 535, Y: 362}, slippy.Tile{Z: 10, X: 535, Y: 363}, slippy.Tile{Z: 10, X: 535, Y: 364}, slippy.Tile{Z: 10, X: 535, Y: 365},
slippy.Tile{Z: 10, X: 536, Y: 356}, slippy.Tile{Z: 10, X: 536, Y: 357}, slippy.Tile{Z: 10, X: 536, Y: 358}, slippy.Tile{Z: 10, X: 536, Y: 359}, slippy.Tile{Z: 10, X: 536, Y: 360}, slippy.Tile{Z: 10, X: 536, Y: 361}, slippy.Tile{Z: 10, X: 536, Y: 362}, slippy.Tile{Z: 10, X: 536, Y: 363}, slippy.Tile{Z: 10, X: 536, Y: 364}, slippy.Tile{Z: 10, X: 536, Y: 365},
slippy.Tile{Z: 10, X: 537, Y: 356}, slippy.Tile{Z: 10, X: 537, Y: 357}, slippy.Tile{Z: 10, X: 537, Y: 358}, slippy.Tile{Z: 10, X: 537, Y: 359}, slippy.Tile{Z: 10, X: 537, Y: 360}, slippy.Tile{Z: 10, X: 537, Y: 361}, slippy.Tile{Z: 10, X: 537, Y: 362}, slippy.Tile{Z: 10, X: 537, Y: 363}, slippy.Tile{Z: 10, X: 537, Y: 364}, slippy.Tile{Z: 10, X: 537, Y: 365},
slippy.Tile{Z: 10, X: 538, Y: 356}, slippy.Tile{Z: 10, X: 538, Y: 357}, slippy.Tile{Z: 10, X: 538, Y: 358}, slippy.Tile{Z: 10, X: 538, Y: 359}, slippy.Tile{Z: 10, X: 538, Y: 360}, slippy.Tile{Z: 10, X: 538, Y: 361}, slippy.Tile{Z: 10, X: 538, Y: 362}, slippy.Tile{Z: 10, X: 538, Y: 363}, slippy.Tile{Z: 10, X: 538, Y: 364}, slippy.Tile{Z: 10, X: 538, Y: 365},
slippy.Tile{Z: 10, X: 539, Y: 356}, slippy.Tile{Z: 10, X: 539, Y: 357}, slippy.Tile{Z: 10, X: 539, Y: 358}, slippy.Tile{Z: 10, X: 539, Y: 359}, slippy.Tile{Z: 10, X: 539, Y: 360}, slippy.Tile{Z: 10, X: 539, Y: 361}, slippy.Tile{Z: 10, X: 539, Y: 362}, slippy.Tile{Z: 10, X: 539, Y: 363}, slippy.Tile{Z: 10, X: 539, Y: 364}, slippy.Tile{Z: 10, X: 539, Y: 365},
slippy.Tile{Z: 10, X: 540, Y: 356}, slippy.Tile{Z: 10, X: 540, Y: 357}, slippy.Tile{Z: 10, X: 540, Y: 358}, slippy.Tile{Z: 10, X: 540, Y: 359}, slippy.Tile{Z: 10, X: 540, Y: 360}, slippy.Tile{Z: 10, X: 540, Y: 361}, slippy.Tile{Z: 10, X: 540, Y: 362}, slippy.Tile{Z: 10, X: 540, Y: 363}, slippy.Tile{Z: 10, X: 540, Y: 364}, slippy.Tile{Z: 10, X: 540, Y: 365},
slippy.Tile{Z: 10, X: 541, Y: 356}, slippy.Tile{Z: 10, X: 541, Y: 357}, slippy.Tile{Z: 10, X: 541, Y: 358}, slippy.Tile{Z: 10, X: 541, Y: 359}, slippy.Tile{Z: 10, X: 541, Y: 360}, slippy.Tile{Z: 10, X: 541, Y: 361}, slippy.Tile{Z: 10, X: 541, Y: 362}, slippy.Tile{Z: 10, X: 541, Y: 363}, slippy.Tile{Z: 10, X: 541, Y: 364}, slippy.Tile{Z: 10, X: 541, Y: 365},
},
},
}

for name, tc := range tests {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/gdey/tbltest v0.0.0-20170331191646-af8abc47b052
github.com/go-spatial/cobra v0.0.3-0.20181105183926-68194e4fbcc6
github.com/go-spatial/geom v0.1.0
github.com/go-spatial/proj v0.2.0
github.com/go-spatial/proj v0.3.0
github.com/go-test/deep v1.1.1
github.com/golang/protobuf v1.5.3
github.com/jackc/pgproto3/v2 v2.3.3
Expand Down Expand Up @@ -65,7 +65,7 @@ require (
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/exp v0.0.0-20230116083435-1de6713980de // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/oauth2 v0.7.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ github.com/go-spatial/geom v0.1.0 h1:IZ6LVz0Kkgd8+U83MmI0J4L4TBDzk5I8xiYe9Ci+aHk
github.com/go-spatial/geom v0.1.0/go.mod h1:yNr22dnkQyFEwq2MJVmmiReiFesQpXKlosAG+qUSPus=
github.com/go-spatial/proj v0.2.0 h1:sii5Now3GFEyR9hV2SBJFWFz95s4xSaZmP0aYDk1K6c=
github.com/go-spatial/proj v0.2.0/go.mod h1:ePHHp7ITVc4eIVW5sgG/0Eu9RMAGOQUgM/D1ZkccY+0=
github.com/go-spatial/proj v0.3.0 h1:2IRb4elpnmVy1tUKRGBR68uiyZU4Pmny8yZC7TEUdSg=
github.com/go-spatial/proj v0.3.0/go.mod h1:wF0V0A60cTD5uzXeg+P52E1GtMY40PIiyYNJVvHXm6k=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
Expand Down Expand Up @@ -296,6 +298,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
Expand Down
56 changes: 45 additions & 11 deletions vendor/github.com/go-spatial/proj/Convert.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 17 additions & 15 deletions vendor/github.com/go-spatial/proj/core/ConvertLPToXY.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions vendor/github.com/go-spatial/proj/core/OperationDescription.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading