diff --git a/cmd/gpq/command/command.go b/cmd/gpq/command/command.go index 53ff7f4..65d0574 100644 --- a/cmd/gpq/command/command.go +++ b/cmd/gpq/command/command.go @@ -15,6 +15,7 @@ var CLI struct { Convert ConvertCmd `cmd:"" help:"Convert data from one format to another."` Validate ValidateCmd `cmd:"" help:"Validate a GeoParquet file."` Describe DescribeCmd `cmd:"" help:"Describe a GeoParquet file."` + Extract ExtractCmd `cmd:"" help:"Extract columns by name or rows by spatial subsetting."` Version VersionCmd `cmd:"" help:"Print the version of this program."` } @@ -49,3 +50,11 @@ func readerFromInput(input string) (storage.ReaderAtSeeker, error) { return os.Open(input) } + +func hasStdin() bool { + stats, err := os.Stdin.Stat() + if err != nil { + return false + } + return stats.Size() > 0 +} diff --git a/cmd/gpq/command/convert.go b/cmd/gpq/command/convert.go index febb425..b200dbe 100644 --- a/cmd/gpq/command/convert.go +++ b/cmd/gpq/command/convert.go @@ -32,9 +32,10 @@ type ConvertCmd struct { To string `help:"Output file format. Possible values: ${enum}." enum:"auto, geojson, geoparquet" default:"auto"` Min int `help:"Minimum number of features to consider when building a schema." default:"10"` Max int `help:"Maximum number of features to consider when building a schema." default:"100"` - InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet withtout metadata." default:"geometry"` + InputPrimaryColumn string `help:"Primary geometry column name when reading Parquet without metadata." default:"geometry"` Compression string `help:"Parquet compression to use. Possible values: ${enum}." enum:"uncompressed, snappy, gzip, brotli, zstd" default:"zstd"` RowGroupLength int `help:"Maximum number of rows per group when writing Parquet."` + AddBbox bool `help:"Compute the bounding box of features where not present in GeoJSON input and write to Parquet output."` } type FormatType string @@ -100,14 +101,6 @@ func getFormatType(resource string) FormatType { return UnknownType } -func hasStdin() bool { - stats, err := os.Stdin.Stat() - if err != nil { - return false - } - return stats.Size() > 0 -} - func (c *ConvertCmd) Run() error { inputSource := c.Input outputSource := c.Output @@ -156,6 +149,10 @@ func (c *ConvertCmd) Run() error { output = o } + if c.AddBbox && outputFormat != GeoParquetType { + return NewCommandError("--add-bbox is only available when converting to GeoParquet.") + } + if inputFormat == GeoJSONType { if outputFormat != ParquetType && outputFormat != GeoParquetType { return NewCommandError("GeoJSON input can only be converted to GeoParquet") @@ -165,6 +162,7 @@ func (c *ConvertCmd) Run() error { MaxFeatures: c.Max, Compression: c.Compression, RowGroupLength: c.RowGroupLength, + AddBbox: c.AddBbox, } if err := geojson.ToParquet(input, output, convertOptions); err != nil { return NewCommandError("%w", err) diff --git a/cmd/gpq/command/extract.go b/cmd/gpq/command/extract.go new file mode 100644 index 0000000..129af7c --- /dev/null +++ b/cmd/gpq/command/extract.go @@ -0,0 +1,181 @@ +package command + +import ( + "context" + "io" + "os" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/planetlabs/gpq/internal/geo" + "github.com/planetlabs/gpq/internal/geoparquet" +) + +type ExtractCmd struct { + Input string `arg:"" optional:"" name:"input" help:"Input file path or URL. If not provided, input is read from stdin."` + Output string `arg:"" optional:"" name:"output" help:"Output file. If not provided, output is written to stdout." type:"path"` + Bbox string `help:"Filter features by intersection of their bounding box with the provided bounding box (in x_min,y_min,x_max,y_max format)."` + DropCols string `help:"Drop the provided columns. Provide a comma-separated string of column names to be excluded. Do not use together with --keep-only-cols."` + KeepOnlyCols string `help:"Keep only the provided columns. Provide a comma-separated string of columns to be kept. Do not use together with --drop-cols."` +} + +func (c *ExtractCmd) Run() error { + + // validate and transform inputs + + inputSource := c.Input + outputSource := c.Output + + if c.Input == "" && hasStdin() { + outputSource = inputSource + inputSource = "" + } + + input, inputErr := readerFromInput(inputSource) + if inputErr != nil { + return NewCommandError("trouble getting a reader from %q: %w", c.Input, inputErr) + } + + var output *os.File + if outputSource == "" { + output = os.Stdout + } else { + o, createErr := os.Create(outputSource) + if createErr != nil { + return NewCommandError("failed to open %q for writing: %w", outputSource, createErr) + } + defer o.Close() + output = o + } + + // prepare input reader (ignore certain columns if asked to - DropCols/KeepOnlyCols) + config := &geoparquet.ReaderConfig{Reader: input} + + parquetFileReader, err := geoparquet.NewParquetFileReader(config) + if err != nil { + return NewCommandError("could not get ParquetFileReader: %w", err) + } + + arrowFileReader, err := geoparquet.NewArrowFileReader(config, parquetFileReader) + if err != nil { + return NewCommandError("could not get ArrowFileReader: %w", err) + } + + geoMetadata, err := geoparquet.GetMetadataFromFileReader(parquetFileReader) + if err != nil { + return NewCommandError("could not get geo metadata from file reader: %w", err) + } + + arrowSchema, schemaErr := arrowFileReader.Schema() + if schemaErr != nil { + return NewCommandError("trouble getting arrow schema: %w", schemaErr) + } + + // projection pushdown - column filtering + var columnIndices []int = nil + + var includeColumns []string + var excludeColumns []string + if c.DropCols != "" { + excludeColumns = strings.Split(c.DropCols, ",") + } + if c.KeepOnlyCols != "" { + includeColumns = strings.Split(c.KeepOnlyCols, ",") + } + + excludeColNamesProvided := len(excludeColumns) > 0 + includeColNamesProvided := len(includeColumns) > 0 + + if excludeColNamesProvided || includeColNamesProvided { + if excludeColNamesProvided == includeColNamesProvided { + return NewCommandError("please pass only one of DropColumns/KeepOnlyColumns") + } + + if includeColNamesProvided { + columnIndices, err = geoparquet.GetColumnIndices(includeColumns, arrowSchema) + if err != nil { + return NewCommandError("trouble inferring column names (positive selection): %w", err) + } + } + + if excludeColNamesProvided { + columnIndices, err = geoparquet.GetColumnIndicesByDifference(excludeColumns, arrowSchema) + if err != nil { + return NewCommandError("trouble inferring column names (negative selection): %w", err) + } + } + } + config.Columns = columnIndices + + // predicate pushdown - spatial row filtering + var rowGroups []int = nil + + // parse bbox filter argument into geo.Bbox struct if applicable + inputBbox, err := geo.NewBboxFromString(c.Bbox) + if err != nil { + return NewCommandError("trouble getting bbox from input string: %w", err) + } + var bboxCol *geoparquet.BboxColumn + if inputBbox != nil { + bboxCol = geoparquet.GetBboxColumn(parquetFileReader.MetaData().Schema, geoMetadata) + + if bboxCol.Name != "" { // if there is a bbox col in the file + rowGroups, err = geoparquet.GetRowGroupsByBbox(parquetFileReader, bboxCol, inputBbox) + if err != nil { + return NewCommandError("trouble scanning row group metadata: %w", err) + } + } + } + + config.RowGroups = rowGroups + + // create new record reader - based on the config values for + // Columns and RowGroups it will only read a subset of + // columns and row groups + ctx := context.Background() + + recordReader, err := geoparquet.NewRecordReader(ctx, arrowFileReader, geoMetadata, columnIndices, rowGroups) + if err != nil { + return NewCommandError("trouble creating geoparquet record reader: %w", err) + } + defer recordReader.Close() + + // prepare output writer + recordWriter, rwErr := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ + Writer: output, + Metadata: recordReader.Metadata(), + ArrowSchema: recordReader.ArrowSchema(), + }) + if rwErr != nil { + return NewCommandError("trouble getting record writer: %w", rwErr) + } + defer recordWriter.Close() + + // read and write records in loop + for { + record, readErr := recordReader.Read() + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + + // filter by bbox if asked to + var filteredRecord *arrow.Record + if inputBbox != nil && bboxCol != nil { + var filterErr error + filteredRecord, filterErr = geoparquet.FilterRecordBatchByBbox(ctx, &record, inputBbox, bboxCol) + if filterErr != nil { + return NewCommandError("trouble filtering record batch by bbox: %w", filterErr) + } + } else { + filteredRecord = &record + } + + if err := recordWriter.Write(*filteredRecord); err != nil { + return err + } + } + return nil +} diff --git a/cmd/gpq/command/extract_test.go b/cmd/gpq/command/extract_test.go new file mode 100644 index 0000000..5f3d459 --- /dev/null +++ b/cmd/gpq/command/extract_test.go @@ -0,0 +1,152 @@ +package command_test + +import ( + "bytes" + + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/planetlabs/gpq/cmd/gpq/command" + "github.com/planetlabs/gpq/internal/geoparquet" +) + +func (s *Suite) TestExtractDropCols() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.0.0.parquet", + DropCols: "pop_est,iso_a3", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + fileReader, err := file.NewParquetReader(bytes.NewReader(data)) + s.Require().NoError(err) + defer fileReader.Close() + + s.Equal(int64(5), fileReader.NumRows()) + + s.Require().NoError(err) + s.Equal(4, fileReader.MetaData().Schema.NumColumns()) + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(4), record.NumCols()) +} + +func (s *Suite) TestExtractKeepOnlyCols() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0.parquet", + KeepOnlyCols: "geometry,pop_est,iso_a3", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + fileReader, err := file.NewParquetReader(bytes.NewReader(data)) + s.Require().NoError(err) + defer fileReader.Close() + + s.Equal(int64(5), fileReader.NumRows()) + + s.Require().NoError(err) + s.Equal(3, fileReader.MetaData().Schema.NumColumns()) + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(3), record.NumCols()) +} + +// Since the 1.1.0 parquet file includes a bbox column, we expect the bbox column to be used for spatial filtering. +func (s *Suite) TestExtractBbox110() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(7), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} + +// Since the 1.1.0 parquet file includes a bbox column and is partitioned into spatially ordered row groups, +// we expect the bbox column row group statistic to be used for spatial pushdown filtering. +func (s *Suite) TestExtractBbox110Partitioned() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.1.0-partitioned.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(8), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} + +// Since the 1.0.0 parquet file doesn't have a bbox column, we expect the bbox column to be calculated on the fly. +func (s *Suite) TestExtractBbox100() { + cmd := &command.ExtractCmd{ + Input: "../../../internal/testdata/cases/example-v1.0.0.parquet", + Bbox: "34,-7,36,-6", + } + s.Require().NoError(cmd.Run()) + + data := s.readStdout() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: bytes.NewReader(data), + }) + s.Require().NoError(err) + defer recordReader.Close() + + // we expect only one row, namely Tanzania + s.Require().Equal(int64(1), recordReader.NumRows()) + + record, readErr := recordReader.Read() + s.Require().NoError(readErr) + s.Assert().Equal(int64(6), record.NumCols()) + s.Assert().Equal(int64(1), record.NumRows()) + + country := record.Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + s.Assert().Equal("Tanzania", country) +} diff --git a/internal/geo/geo.go b/internal/geo/geo.go index f560dbe..f6a263e 100644 --- a/internal/geo/geo.go +++ b/internal/geo/geo.go @@ -2,8 +2,11 @@ package geo import ( "encoding/json" + "errors" "fmt" "math" + "strconv" + "strings" "sync" "github.com/paulmach/orb" @@ -33,6 +36,7 @@ type Feature struct { Id any `json:"id,omitempty"` Type string `json:"type"` Geometry orb.Geometry `json:"geometry"` + Bbox *orb.Bound `json:"bbox,omitempty"` Properties map[string]any `json:"properties"` } @@ -50,12 +54,16 @@ func (f *Feature) MarshalJSON() ([]byte, error) { if f.Id != nil { m["id"] = f.Id } + if f.Bbox != nil { + m["bbox"] = f.Bbox + } return json.Marshal(m) } type jsonFeature struct { Id any `json:"id,omitempty"` Type string `json:"type"` + Bbox *orbjson.BBox `json:"bbox,omitempty"` Geometry json.RawMessage `json:"geometry"` Properties map[string]any `json:"properties"` } @@ -93,6 +101,14 @@ func (f *Feature) UnmarshalJSON(data []byte) error { } f.Geometry = geometry.Geometry() + + if jf.Bbox != nil { + if !jf.Bbox.Valid() { + return errors.New("invalid bbox, make sure it is an array of at least 4 floats") + } + bound := jf.Bbox.Bound() + f.Bbox = &bound + } return nil } @@ -334,3 +350,77 @@ func (i *DatasetStats) Types(name string) []string { i.readUnlock() return collection.Types() } + +// BBOX type + +type Bbox struct { + Xmin float64 `parquet:"name=xmin" json:"xmin"` + Ymin float64 `parquet:"name=ymin" json:"ymin"` + Xmax float64 `parquet:"name=xmax" json:"xmax"` + Ymax float64 `parquet:"name=ymax" json:"ymax"` +} + +// Checks whether the bbox overlaps with another axis-aligned bbox. +func (box1 *Bbox) Intersects(box2 *Bbox) bool { + // check latitude overlap + if box1.Ymax < box2.Ymin || box2.Ymax < box1.Ymin { + return false + } + + // if box1 crosses the antimeridian and uses the coordinate range -180/180, + // represent e.g. xmin 170 as -190 + if box1.Xmin > 0 && box1.Xmax < 0 { + box1.Xmin = -180 - (180 - box1.Xmin) + } + + // see above + if box2.Xmin > 0 && box2.Xmax < 0 { + box2.Xmin = -180 - (180 - box2.Xmin) + } + + // check longitude overlap + if box1.Xmax < box2.Xmin || box2.Xmax < box1.Xmin { + return false + } + + return true +} + +// Create a new Bbox struct from a string of comma-separated values in format xmin,ymin,xmax,ymax. +func NewBboxFromString(bounds string) (*Bbox, error) { + inputBbox := &Bbox{} + + if bounds != "" { + bboxValues := strings.Split(bounds, ",") + if len(bboxValues) != 4 { + return nil, errors.New("please provide 4 comma-separated values (xmin,ymin,xmax,ymax) as a bbox") + } + + xminInput, err := strconv.ParseFloat(bboxValues[0], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing xmin input as float64: %w", err) + } + inputBbox.Xmin = xminInput + + yminInput, err := strconv.ParseFloat(bboxValues[1], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing ymin input as float64: %w", err) + } + inputBbox.Ymin = yminInput + + xmaxInput, err := strconv.ParseFloat(bboxValues[2], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing xmax input as float64: %w", err) + } + inputBbox.Xmax = xmaxInput + + ymaxInput, err := strconv.ParseFloat(bboxValues[3], 64) + if err != nil { + return nil, fmt.Errorf("trouble parsing ymax input as float64: %w", err) + } + inputBbox.Ymax = ymaxInput + } else { + inputBbox = nil + } + return inputBbox, nil +} diff --git a/internal/geo/geo_test.go b/internal/geo/geo_test.go new file mode 100644 index 0000000..eb341f1 --- /dev/null +++ b/internal/geo/geo_test.go @@ -0,0 +1,155 @@ +package geo + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBboxIntersectsTrue(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 20, + Xmax: 30, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: 25, + Ymin: 35, + Xmax: 45, + Ymax: 55, + } + + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsFalse(t *testing.T) { + box1 := &Bbox{ + Xmin: -10, + Ymin: 20, + Xmax: -5, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: -1, + Ymin: 50, + Xmax: 0, + Ymax: 70, + } + + require.Equal(t, false, box1.Intersects(box2)) +} + +func TestBboxIntersectsTouches(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 20, + Xmax: 30, + Ymax: 40, + } + + box2 := &Bbox{ + Xmin: 30, + Ymin: 20, + Xmax: 40, + Ymax: 40, + } + + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsWholeGlobe(t *testing.T) { + box1 := &Bbox{ + Xmin: -180, + Ymin: -90, + Xmax: 180, + Ymax: 90, + } + + box2 := &Bbox{ + Xmin: 10, + Ymin: 10, + Xmax: 30, + Ymax: 30, + } + + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsContains(t *testing.T) { + box1 := &Bbox{ + Xmin: 10, + Ymin: 10, + Xmax: 30, + Ymax: 30, + } + + box2 := &Bbox{ + Xmin: 0, + Ymin: 0, + Xmax: 40, + Ymax: 40, + } + + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsTrueAntimeridian(t *testing.T) { + box1 := &Bbox{ + Xmin: 170, + Ymin: -10, + Xmax: -165, + Ymax: 10, + } + + box2 := &Bbox{ + Xmin: -180, + Ymin: -5, + Xmax: -170, + Ymax: 15, + } + + require.Equal(t, true, box1.Intersects(box2)) +} + +func TestBboxIntersectsFalseAntimeridian(t *testing.T) { + box1 := &Bbox{ + Xmin: 170, + Ymin: -10, + Xmax: 180, + Ymax: 10, + } + + box2 := &Bbox{ + Xmin: -160, + Ymin: -5, + Xmax: -150, + Ymax: 15, + } + + require.Equal(t, false, box1.Intersects(box2)) +} + +func TestNewBboxFromString(t *testing.T) { + bbox, err := NewBboxFromString("-160,-5,-150,15") + assert.NoError(t, err) + assert.Equal(t, -160.0, bbox.Xmin) + assert.Equal(t, -5.0, bbox.Ymin) + assert.Equal(t, -150.0, bbox.Xmax) + assert.Equal(t, 15.0, bbox.Ymax) +} + +func TestNewBboxFromStringErrNotEnoughValues(t *testing.T) { + bbox, err := NewBboxFromString("-160,-5,-150") + assert.ErrorContains(t, err, "please provide 4") + assert.Nil(t, bbox) +} + +func TestNewBboxFromStringErrWrongType(t *testing.T) { + bbox, err := NewBboxFromString("foo,-5,-150,15") + assert.ErrorContains(t, err, "float") + assert.Nil(t, bbox) +} diff --git a/internal/geojson/featurereader.go b/internal/geojson/featurereader.go index af64571..a04832a 100644 --- a/internal/geojson/featurereader.go +++ b/internal/geojson/featurereader.go @@ -95,6 +95,24 @@ func (r *FeatureReader) Read() (*geo.Feature, error) { continue } + if key == "bbox" { + if feature == nil { + feature = &geo.Feature{} + } else if feature.Bbox != nil { + return nil, errors.New("found duplicate bbox") + } + bbox := &orbjson.BBox{} + if err := r.decoder.Decode(bbox); err != nil { + return nil, fmt.Errorf("trouble parsing bbox: %w", err) + } + if !bbox.Valid() { + return nil, errors.New("invalid bbox, make sure it is an array of at least 4 floats") + } + bound := bbox.Bound() + feature.Bbox = &bound + continue + } + if key == "properties" { if feature == nil { feature = &geo.Feature{} diff --git a/internal/geojson/featurereader_test.go b/internal/geojson/featurereader_test.go index 31f4ed4..1c0cc4d 100644 --- a/internal/geojson/featurereader_test.go +++ b/internal/geojson/featurereader_test.go @@ -120,6 +120,49 @@ func TestFeatureReaderNewLineDelimited(t *testing.T) { assert.Equal(t, float64(326625791), usa.Properties["pop_est"]) } +func TestFeatureReaderBbox(t *testing.T) { + file, openErr := os.Open("testdata/bbox.geojson") + require.NoError(t, openErr) + + reader := geojson.NewFeatureReader(file) + + features := []*geo.Feature{} + for { + feature, err := reader.Read() + if err == io.EOF { + break + } + require.NoError(t, err) + features = append(features, feature) + } + require.Len(t, features, 5) + + feature := features[0] + require.NotNil(t, feature.Geometry) + assert.Equal(t, "MultiPolygon", feature.Geometry.GeoJSONType()) + assert.Equal(t, map[string]any{ + "continent": "Oceania", + "gdp_md_est": 8374.0, + "iso_a3": "FJI", + "name": "Fiji", + "pop_est": 920938.0}, feature.Properties) + assert.Equal(t, -180.0, feature.Bbox.Min.X()) + assert.Equal(t, -18.28799, feature.Bbox.Min.Y()) + assert.Equal(t, 180.0, feature.Bbox.Max.X()) + assert.Equal(t, -16.020882256741224, feature.Bbox.Max.Y()) +} + +func TestFeatureReaderBboxInvalid(t *testing.T) { + file, openErr := os.Open("testdata/bad-bbox.geojson") + require.NoError(t, openErr) + + reader := geojson.NewFeatureReader(file) + + feature, err := reader.Read() + require.ErrorContains(t, err, "invalid bbox") + require.Nil(t, feature) +} + func TestFeatureReaderBadNewLineDelimited(t *testing.T) { file, openErr := os.Open("testdata/bad-new-line-delimited.ndgeojson") require.NoError(t, openErr) diff --git a/internal/geojson/geojson.go b/internal/geojson/geojson.go index 73978d1..8a7d7db 100644 --- a/internal/geojson/geojson.go +++ b/internal/geojson/geojson.go @@ -10,23 +10,8 @@ import ( "github.com/planetlabs/gpq/internal/pqutil" ) -const primaryColumn = "geometry" - -func GetDefaultMetadata() *geoparquet.Metadata { - return &geoparquet.Metadata{ - Version: geoparquet.Version, - PrimaryColumn: primaryColumn, - Columns: map[string]*geoparquet.GeometryColumn{ - primaryColumn: { - Encoding: "WKB", - GeometryTypes: []string{}, - }, - }, - } -} - func FromParquet(reader parquet.ReaderAtSeeker, writer io.Writer) error { - recordReader, rrErr := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, rrErr := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: reader, }) if rrErr != nil { @@ -63,6 +48,7 @@ type ConvertOptions struct { Compression string RowGroupLength int Metadata string + AddBbox bool } var defaultOptions = &ConvertOptions{ @@ -96,6 +82,8 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions pqWriterProps = parquet.NewWriterProperties(writerOptions...) } + writeCoveringMetadata := convertOptions.AddBbox + var featureWriter *geoparquet.FeatureWriter writeBuffered := func() error { if !builder.Ready() { @@ -109,9 +97,10 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions return scErr } fw, fwErr := geoparquet.NewFeatureWriter(&geoparquet.WriterConfig{ - Writer: output, - ArrowSchema: sc, - ParquetWriterProps: pqWriterProps, + Writer: output, + ArrowSchema: sc, + ParquetWriterProps: pqWriterProps, + WriteCoveringMetadata: writeCoveringMetadata, }) if fwErr != nil { return fwErr @@ -135,11 +124,25 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions return err } featuresRead += 1 + + if feature.Bbox == nil && convertOptions.AddBbox { + bound := feature.Geometry.Bound() + feature.Bbox = &bound + } + + if feature.Bbox != nil { + writeCoveringMetadata = true + } + if featureWriter == nil { if err := builder.Add(feature.Properties); err != nil { return err } + if feature.Bbox != nil { + builder.AddBbox(geoparquet.DefaultBboxColumn) + } + if !builder.Ready() { buffer = append(buffer, feature) if len(buffer) > convertOptions.MaxFeatures { diff --git a/internal/geojson/geojson_test.go b/internal/geojson/geojson_test.go index 852c4bf..86e547c 100644 --- a/internal/geojson/geojson_test.go +++ b/internal/geojson/geojson_test.go @@ -17,6 +17,7 @@ package geojson_test import ( "bytes" "encoding/json" + "fmt" "os" "strings" "testing" @@ -83,6 +84,12 @@ func TestToParquet(t *testing.T) { metadata, geoErr := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) require.NoError(t, geoErr) + assert.Equal(t, "geometry", metadata.PrimaryColumn) + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + assert.Nil(t, metadata.Columns[metadata.PrimaryColumn].Covering) geometryTypes := metadata.Columns[metadata.PrimaryColumn].GetGeometryTypes() assert.Len(t, geometryTypes, 2) assert.Contains(t, geometryTypes, "MultiPolygon") @@ -273,6 +280,78 @@ func TestToParquetNumberId(t *testing.T) { assert.Equal(t, []string{"Point"}, geometryTypes) } +func TestToParquetExistingBbox(t *testing.T) { + geojsonFile, openErr := os.Open("testdata/bbox.geojson") + require.NoError(t, openErr) + + parquetBuffer := &bytes.Buffer{} + toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, nil) + assert.NoError(t, toParquetErr) + + parquetInput := bytes.NewReader(parquetBuffer.Bytes()) + fileReader, fileErr := file.NewParquetReader(parquetInput) + require.NoError(t, fileErr) + defer fileReader.Close() + + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + require.NotNil(t, metadata.Columns[metadata.PrimaryColumn].Covering) + assert.Equal(t, []string{"bbox", "xmin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) + assert.Equal(t, []string{"bbox", "ymin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin) + assert.Equal(t, []string{"bbox", "xmax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax) + assert.Equal(t, []string{"bbox", "ymax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax) + + assert.NotEqual(t, -1, fileReader.MetaData().Schema.ColumnIndexByName("bbox.xmin")) + assert.Equal(t, int64(5), fileReader.NumRows()) + + geojsonBuffer := &bytes.Buffer{} + fromParquetErr := geojson.FromParquet(parquetInput, geojsonBuffer) + require.NoError(t, fromParquetErr) + + expected, err := os.ReadFile("testdata/bbox.geojson") + require.NoError(t, err) + + assert.JSONEq(t, string(expected), geojsonBuffer.String()) +} + +func TestToParquetAddBbox(t *testing.T) { + geojsonFile, openErr := os.Open("testdata/example.geojson") + require.NoError(t, openErr) + + parquetBuffer := &bytes.Buffer{} + toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, &geojson.ConvertOptions{AddBbox: true}) + assert.NoError(t, toParquetErr) + + parquetInput := bytes.NewReader(parquetBuffer.Bytes()) + fileReader, fileErr := file.NewParquetReader(parquetInput) + require.NoError(t, fileErr) + defer fileReader.Close() + + assert.NotEqual(t, -1, fileReader.MetaData().Schema.ColumnIndexByName("bbox.xmin")) + assert.Equal(t, int64(5), fileReader.NumRows()) + + // check if covering metadata has been written + metadata, err := geoparquet.GetMetadata(fileReader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + require.NotNil(t, metadata.Columns[metadata.PrimaryColumn].Covering) + assert.Equal(t, []string{"bbox", "xmin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin) + assert.Equal(t, []string{"bbox", "ymin"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin) + assert.Equal(t, []string{"bbox", "xmax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax) + assert.Equal(t, []string{"bbox", "ymax"}, metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax) + + geojsonBuffer := &bytes.Buffer{} + fromParquetErr := geojson.FromParquet(parquetInput, geojsonBuffer) + require.NoError(t, fromParquetErr) + + expected, err := os.ReadFile("testdata/bbox.geojson") + require.NoError(t, err) + + assert.JSONEq(t, string(expected), geojsonBuffer.String()) +} + func TestToParquetBooleanId(t *testing.T) { geojsonFile, openErr := os.Open("testdata/boolean-id.geojson") require.NoError(t, openErr) @@ -457,7 +536,7 @@ func TestRoundTripSparseProperties(t *testing.T) { assert.JSONEq(t, string(inputData), jsonBuffer.String()) } -func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*bytes.Reader, error) { +func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata, writeCoveringMetadata bool) (*bytes.Reader, error) { data, err := json.Marshal(rows) if err != nil { return nil, err @@ -465,19 +544,20 @@ func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*byte parquetSchema, err := schema.NewSchemaFromStruct(rows[0]) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create parquet schema from struct: %w", err) } arrowSchema, err := pqarrow.FromParquet(parquetSchema, nil, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create arrow schema from struct: %w", err) } output := &bytes.Buffer{} recordWriter, err := geoparquet.NewRecordWriter(&geoparquet.WriterConfig{ - Writer: output, - Metadata: metadata, - ArrowSchema: arrowSchema, + Writer: output, + Metadata: metadata, + ArrowSchema: arrowSchema, + WriteCoveringMetadata: writeCoveringMetadata, }) if err != nil { return nil, err @@ -485,7 +565,7 @@ func makeGeoParquetReader[T any](rows []T, metadata *geoparquet.Metadata) (*byte rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSchema, strings.NewReader(string(data))) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot create record from json: %w", err) } if err := recordWriter.Write(rec); err != nil { @@ -515,10 +595,10 @@ func TestWKT(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = geo.EncodingWKT - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -567,10 +647,10 @@ func TestWKTNoEncoding(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = "" - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -612,9 +692,9 @@ func TestWKB(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -656,10 +736,10 @@ func TestWKBNoEncoding(t *testing.T) { }, } - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) metadata.Columns[metadata.PrimaryColumn].Encoding = "" - reader, readerErr := makeGeoParquetReader(rows, metadata) + reader, readerErr := makeGeoParquetReader(rows, metadata, false) require.NoError(t, readerErr) output := &bytes.Buffer{} @@ -774,3 +854,48 @@ func TestCodecInvalid(t *testing.T) { toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions) assert.EqualError(t, toParquetErr, "invalid compression codec invalid") } + +func TestCoveringMetadata(t *testing.T) { + type Row struct { + Name string `parquet:"name=name, logical=String" json:"name"` + Geometry string `parquet:"name=geometry, logical=String" json:"geometry"` + Bbox geo.Bbox `parquet:"name=bbox" json:"bbox"` + } + + rows := []*Row{ + { + Name: "test-point", + Geometry: "POINT (1 2)", + Bbox: geo.Bbox{Xmin: 1, Ymin: 2, Xmax: 1, Ymax: 2}, + }, + } + + metadata := geoparquet.DefaultMetadata(true) + metadata.Columns[metadata.PrimaryColumn].Encoding = geo.EncodingWKT + + reader, readerErr := makeGeoParquetReader(rows, metadata, true) + require.NoError(t, readerErr) + + output := &bytes.Buffer{} + convertErr := geojson.FromParquet(reader, output) + require.NoError(t, convertErr) + + expected := `{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "bbox": [1, 2, 1, 2], + "properties": { + "name": "test-point" + }, + "geometry": { + "type": "Point", + "coordinates": [1, 2] + } + } + ] + }` + + assert.JSONEq(t, expected, output.String()) +} diff --git a/internal/geojson/recordwriter.go b/internal/geojson/recordwriter.go index 69a67dc..52d6488 100644 --- a/internal/geojson/recordwriter.go +++ b/internal/geojson/recordwriter.go @@ -2,6 +2,8 @@ package geojson import ( "encoding/json" + "errors" + "fmt" "io" "github.com/apache/arrow/go/v16/arrow" @@ -51,6 +53,7 @@ func (w *RecordWriter) Write(record arrow.Record) error { } var geometry *orbjson.Geometry + var bbox *orbjson.BBox properties := map[string]any{} for fieldNum := 0; fieldNum < arr.NumField(); fieldNum += 1 { value := arr.Field(fieldNum).GetOneForMarshal(rowNum) @@ -67,6 +70,29 @@ func (w *RecordWriter) Write(record arrow.Record) error { properties[name] = g continue } + + bboxCol := geoparquet.GetBboxColumnNameFromMetadata(w.geoMetadata) + if value != nil && (name == bboxCol || (bboxCol == "" && name == geoparquet.DefaultBboxColumn)) { + bboxMap, ok := value.(map[string]any) + if !ok { + return errors.New("value is not of type map[string]any") + } + fieldNames := geoparquet.GetBboxColumnFieldNames(w.geoMetadata) + xmin, xminOk := bboxMap[fieldNames.Xmin] + ymin, yminOk := bboxMap[fieldNames.Ymin] + xmax, xmaxOk := bboxMap[fieldNames.Xmax] + ymax, ymaxOk := bboxMap[fieldNames.Ymax] + if !(xminOk && yminOk && xmaxOk && ymaxOk) { + return fmt.Errorf("bbox struct must have fields %v/%v/%v/%v", fieldNames.Xmin, fieldNames.Ymin, fieldNames.Xmax, fieldNames.Ymax) + } + if xmin == nil || ymin == nil || xmax == nil || ymax == nil { + return errors.New("bbox struct must have non-null values") + } + orbBbox := orbjson.BBox([]float64{xmin.(float64), ymin.(float64), xmax.(float64), ymax.(float64)}) + bbox = &orbBbox + continue + } + properties[name] = value } @@ -76,6 +102,10 @@ func (w *RecordWriter) Write(record arrow.Record) error { "geometry": geometry, } + if bbox != nil { + feature["bbox"] = bbox + } + featureData, jsonErr := json.Marshal(feature) if jsonErr != nil { return jsonErr diff --git a/internal/geojson/testdata/bad-bbox.geojson b/internal/geojson/testdata/bad-bbox.geojson new file mode 100644 index 0000000..3670651 --- /dev/null +++ b/internal/geojson/testdata/bad-bbox.geojson @@ -0,0 +1,135 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "bbox": [ + -17.06342322434257, + 20.999752102130827, + -8.665124477564191 + ], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.665589565454809, + 27.656425889592356 + ], + [ + -8.665124477564191, + 27.589479071558227 + ], + [ + -8.684399786809053, + 27.395744126896005 + ], + [ + -8.6872936670174, + 25.881056219988906 + ], + [ + -11.96941891117116, + 25.933352769468268 + ], + [ + -11.937224493853321, + 23.374594224536168 + ], + [ + -12.874221564169575, + 23.284832261645178 + ], + [ + -13.118754441774712, + 22.771220201096256 + ], + [ + -12.929101935263532, + 21.327070624267563 + ], + [ + -16.845193650773993, + 21.33332347257488 + ], + [ + -17.06342322434257, + 20.999752102130827 + ], + [ + -17.02042843267577, + 21.422310288981578 + ], + [ + -17.00296179856109, + 21.420734157796577 + ], + [ + -14.750954555713534, + 21.500600083903663 + ], + [ + -14.630832688851072, + 21.860939846274903 + ], + [ + -14.221167771857253, + 22.31016307218816 + ], + [ + -13.891110398809047, + 23.691009019459305 + ], + [ + -12.50096269372537, + 24.7701162785782 + ], + [ + -12.03075883630163, + 26.030866197203068 + ], + [ + -11.718219773800357, + 26.104091701760623 + ], + [ + -11.392554897497007, + 26.883423977154393 + ], + [ + -10.551262579785273, + 26.990807603456886 + ], + [ + -10.189424200877582, + 26.860944729107405 + ], + [ + -9.735343390328879, + 26.860944729107405 + ], + [ + -9.41303748212448, + 27.088476060488574 + ], + [ + -8.794883999049077, + 27.120696316022507 + ], + [ + -8.817828334986672, + 27.656425889592356 + ], + [ + -8.665589565454809, + 27.656425889592356 + ] + ] + ] + }, + "properties": { + "continent": "Africa" + }, + "type": "Feature" + } + ] +} \ No newline at end of file diff --git a/internal/geojson/testdata/bbox.geojson b/internal/geojson/testdata/bbox.geojson new file mode 100644 index 0000000..aa2797f --- /dev/null +++ b/internal/geojson/testdata/bbox.geojson @@ -0,0 +1,5658 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 180, + -16.067132663642447 + ], + [ + 180, + -16.555216566639196 + ], + [ + 179.36414266196414, + -16.801354076946883 + ], + [ + 178.72505936299711, + -17.01204167436804 + ], + [ + 178.59683859511713, + -16.639150000000004 + ], + [ + 179.0966093629971, + -16.433984277547403 + ], + [ + 179.4135093629971, + -16.379054277547404 + ], + [ + 180, + -16.067132663642447 + ] + ] + ], + [ + [ + [ + 178.12557, + -17.50481 + ], + [ + 178.3736, + -17.33992 + ], + [ + 178.71806, + -17.62846 + ], + [ + 178.55271, + -18.15059 + ], + [ + 177.93266000000003, + -18.28799 + ], + [ + 177.38146, + -18.16432 + ], + [ + 177.28504, + -17.72465 + ], + [ + 177.67087, + -17.381140000000002 + ], + [ + 178.12557, + -17.50481 + ] + ] + ], + [ + [ + [ + -179.79332010904864, + -16.020882256741224 + ], + [ + -179.9173693847653, + -16.501783135649397 + ], + [ + -180, + -16.555216566639196 + ], + [ + -180, + -16.067132663642447 + ], + [ + -179.79332010904864, + -16.020882256741224 + ] + ] + ] + ] + }, + "properties": { + "continent": "Oceania", + "gdp_md_est": 8374, + "iso_a3": "FJI", + "name": "Fiji", + "pop_est": 920938 + }, + "type": "Feature", + "bbox": [ + -180.0, + -18.28799, + 180.0, + -16.020882256741224 + ] + }, + { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 33.90371119710453, + -0.9500000000000001 + ], + [ + 34.07261999999997, + -1.0598199999999451 + ], + [ + 37.69868999999994, + -3.0969899999999484 + ], + [ + 37.7669, + -3.6771200000000004 + ], + [ + 39.20222, + -4.67677 + ], + [ + 38.74053999999995, + -5.9089499999999475 + ], + [ + 38.79977000000008, + -6.475660000000005 + ], + [ + 39.44, + -6.839999999999861 + ], + [ + 39.47000000000014, + -7.099999999999966 + ], + [ + 39.19468999999998, + -7.703899999999976 + ], + [ + 39.25203000000005, + -8.00780999999995 + ], + [ + 39.18652000000009, + -8.48550999999992 + ], + [ + 39.53574000000009, + -9.112369999999885 + ], + [ + 39.94960000000003, + -10.098400000000026 + ], + [ + 40.316586229110854, + -10.317097752817492 + ], + [ + 40.31659000000002, + -10.317099999999868 + ], + [ + 39.52099999999996, + -10.89688000000001 + ], + [ + 38.42755659358775, + -11.285202325081656 + ], + [ + 37.827639999999974, + -11.26878999999991 + ], + [ + 37.471289999999954, + -11.568759999999997 + ], + [ + 36.775150994622805, + -11.594537448780805 + ], + [ + 36.51408165868426, + -11.720938002166735 + ], + [ + 35.31239790216904, + -11.439146416879147 + ], + [ + 34.55998904799935, + -11.520020033415925 + ], + [ + 34.27999999999997, + -10.160000000000025 + ], + [ + 33.940837724096525, + -9.693673841980285 + ], + [ + 33.73972000000009, + -9.417149999999992 + ], + [ + 32.75937544122132, + -9.23059905358906 + ], + [ + 32.19186486179194, + -8.930358981973257 + ], + [ + 31.556348097466497, + -8.762048841998642 + ], + [ + 31.15775133695005, + -8.594578747317366 + ], + [ + 30.740009731422095, + -8.34000593035372 + ], + [ + 30.74001549655179, + -8.340007419470915 + ], + [ + 30.199996779101696, + -7.079980970898163 + ], + [ + 29.620032179490014, + -6.520015150583426 + ], + [ + 29.419992710088167, + -5.939998874539434 + ], + [ + 29.519986606572928, + -5.419978936386315 + ], + [ + 29.339997592900346, + -4.4999834122940925 + ], + [ + 29.753512404099865, + -4.452389418153302 + ], + [ + 30.11632000000003, + -4.090120000000013 + ], + [ + 30.505539999999996, + -3.5685799999999404 + ], + [ + 30.752240000000086, + -3.3593099999999936 + ], + [ + 30.743010000000027, + -3.034309999999948 + ], + [ + 30.527660000000026, + -2.807619999999986 + ], + [ + 30.469673645761223, + -2.41385475710134 + ], + [ + 30.469670000000008, + -2.4138299999999617 + ], + [ + 30.75830895358311, + -2.2872502579883687 + ], + [ + 30.816134881317712, + -1.6989140763453887 + ], + [ + 30.419104852019245, + -1.1346591121504161 + ], + [ + 30.769860000000108, + -1.0145499999999856 + ], + [ + 31.866170000000068, + -1.0273599999999306 + ], + [ + 33.90371119710453, + -0.9500000000000001 + ] + ] + ] + }, + "properties": { + "continent": "Africa", + "gdp_md_est": 150600, + "iso_a3": "TZA", + "name": "Tanzania", + "pop_est": 53950935 + }, + "type": "Feature", + "bbox": [ + 29.339997592900346, + -11.720938002166735, + 40.31659000000002, + -0.9500000000000001 + ] + }, + { + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -8.665589565454809, + 27.656425889592356 + ], + [ + -8.665124477564191, + 27.589479071558227 + ], + [ + -8.684399786809053, + 27.395744126896005 + ], + [ + -8.6872936670174, + 25.881056219988906 + ], + [ + -11.96941891117116, + 25.933352769468268 + ], + [ + -11.937224493853321, + 23.374594224536168 + ], + [ + -12.874221564169575, + 23.284832261645178 + ], + [ + -13.118754441774712, + 22.771220201096256 + ], + [ + -12.929101935263532, + 21.327070624267563 + ], + [ + -16.845193650773993, + 21.33332347257488 + ], + [ + -17.06342322434257, + 20.999752102130827 + ], + [ + -17.02042843267577, + 21.422310288981578 + ], + [ + -17.00296179856109, + 21.420734157796577 + ], + [ + -14.750954555713534, + 21.500600083903663 + ], + [ + -14.630832688851072, + 21.860939846274903 + ], + [ + -14.221167771857253, + 22.31016307218816 + ], + [ + -13.891110398809047, + 23.691009019459305 + ], + [ + -12.50096269372537, + 24.7701162785782 + ], + [ + -12.03075883630163, + 26.030866197203068 + ], + [ + -11.718219773800357, + 26.104091701760623 + ], + [ + -11.392554897497007, + 26.883423977154393 + ], + [ + -10.551262579785273, + 26.990807603456886 + ], + [ + -10.189424200877582, + 26.860944729107405 + ], + [ + -9.735343390328879, + 26.860944729107405 + ], + [ + -9.41303748212448, + 27.088476060488574 + ], + [ + -8.794883999049077, + 27.120696316022507 + ], + [ + -8.817828334986672, + 27.656425889592356 + ], + [ + -8.665589565454809, + 27.656425889592356 + ] + ] + ] + }, + "properties": { + "continent": "Africa", + "gdp_md_est": 906.5, + "iso_a3": "ESH", + "name": "W. Sahara", + "pop_est": 603253 + }, + "type": "Feature", + "bbox": [ + -17.06342322434257, + 20.999752102130827, + -8.665124477564191, + 27.656425889592356 + ] + }, + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -122.84000000000003, + 49.000000000000114 + ], + [ + -122.97421000000001, + 49.00253777777778 + ], + [ + -124.91024, + 49.98456 + ], + [ + -125.62461, + 50.416560000000004 + ], + [ + -127.43561000000001, + 50.83061 + ], + [ + -127.99276, + 51.71583 + ], + [ + -127.85032, + 52.32961 + ], + [ + -129.12979, + 52.75538 + ], + [ + -129.30523, + 53.561589999999995 + ], + [ + -130.51497, + 54.28757 + ], + [ + -130.53610895273684, + 54.80275447679924 + ], + [ + -130.53611, + 54.802780000000006 + ], + [ + -129.98, + 55.285000000000004 + ], + [ + -130.00778000000003, + 55.915830000000085 + ], + [ + -131.70781, + 56.55212 + ], + [ + -132.73042, + 57.692890000000006 + ], + [ + -133.35556000000003, + 58.41028000000001 + ], + [ + -134.27111000000002, + 58.86111000000005 + ], + [ + -134.94500000000005, + 59.2705600000001 + ], + [ + -135.47583, + 59.787780000000005 + ], + [ + -136.47972000000004, + 59.46389000000005 + ], + [ + -137.4525, + 58.905 + ], + [ + -138.34089, + 59.562110000000004 + ], + [ + -139.03900000000002, + 60 + ], + [ + -140.013, + 60.27682000000001 + ], + [ + -140.99778, + 60.30639000000001 + ], + [ + -140.9925, + 66.00003000000001 + ], + [ + -140.986, + 69.712 + ], + [ + -140.98598761037601, + 69.71199839952635 + ], + [ + -139.12052, + 69.47102 + ], + [ + -137.54636000000002, + 68.99002 + ], + [ + -136.50358, + 68.89804 + ], + [ + -135.62576, + 69.31512000000001 + ], + [ + -134.41464000000002, + 69.62743 + ], + [ + -132.92925000000002, + 69.50534 + ], + [ + -131.43135999999998, + 69.94451 + ], + [ + -129.79471, + 70.19369 + ], + [ + -129.10773, + 69.77927000000001 + ], + [ + -128.36156, + 70.01286 + ], + [ + -128.13817, + 70.48384 + ], + [ + -127.44712000000001, + 70.37721 + ], + [ + -125.75632000000002, + 69.48058 + ], + [ + -124.42483, + 70.1584 + ], + [ + -124.28968, + 69.39968999999999 + ], + [ + -123.06108, + 69.56372 + ], + [ + -122.6835, + 69.85553 + ], + [ + -121.47226, + 69.79778 + ], + [ + -119.94288, + 69.37786 + ], + [ + -117.60268, + 69.01128 + ], + [ + -116.22643, + 68.84151 + ], + [ + -115.24690000000001, + 68.90591 + ], + [ + -113.89793999999999, + 68.3989 + ], + [ + -115.30489, + 67.90261000000001 + ], + [ + -113.49727, + 67.68815000000001 + ], + [ + -110.798, + 67.80611999999999 + ], + [ + -109.94619, + 67.98104000000001 + ], + [ + -108.8802, + 67.38144 + ], + [ + -107.79239, + 67.88736 + ], + [ + -108.81299, + 68.31164 + ], + [ + -108.16721000000001, + 68.65392 + ], + [ + -106.95, + 68.7 + ], + [ + -106.15, + 68.8 + ], + [ + -105.34282000000002, + 68.56122 + ], + [ + -104.33791000000001, + 68.018 + ], + [ + -103.22115000000001, + 68.09775 + ], + [ + -101.45433, + 67.64689 + ], + [ + -99.90195, + 67.80566 + ], + [ + -98.4432, + 67.78165 + ], + [ + -98.5586, + 68.40394 + ], + [ + -97.66948000000001, + 68.57864000000001 + ], + [ + -96.11991, + 68.23939 + ], + [ + -96.12588, + 67.29338 + ], + [ + -95.48943, + 68.0907 + ], + [ + -94.685, + 68.06383 + ], + [ + -94.23282000000002, + 69.06903000000001 + ], + [ + -95.30408, + 69.68571 + ], + [ + -96.47131, + 70.08976 + ], + [ + -96.39115, + 71.19482 + ], + [ + -95.2088, + 71.92053 + ], + [ + -93.88997, + 71.76015 + ], + [ + -92.87818, + 71.31869 + ], + [ + -91.51964000000001, + 70.19129000000001 + ], + [ + -92.40692000000001, + 69.69997000000001 + ], + [ + -90.5471, + 69.49766 + ], + [ + -90.55151000000001, + 68.47499 + ], + [ + -89.21515, + 69.25873 + ], + [ + -88.01966, + 68.61508 + ], + [ + -88.31748999999999, + 67.87338000000001 + ], + [ + -87.35017, + 67.19872 + ], + [ + -86.30606999999999, + 67.92146 + ], + [ + -85.57664, + 68.78456 + ], + [ + -85.52197, + 69.88211 + ], + [ + -84.10081000000001, + 69.80539 + ], + [ + -82.62258, + 69.65826 + ], + [ + -81.28043000000001, + 69.16202000000001 + ], + [ + -81.22019999999999, + 68.66567 + ], + [ + -81.96436000000001, + 68.13253 + ], + [ + -81.25928, + 67.59716 + ], + [ + -81.38653000000001, + 67.11078 + ], + [ + -83.34456, + 66.41154 + ], + [ + -84.73542, + 66.2573 + ], + [ + -85.76943, + 66.55833 + ], + [ + -86.06760000000001, + 66.05625 + ], + [ + -87.03143, + 65.21297 + ], + [ + -87.32324, + 64.77563 + ], + [ + -88.48296, + 64.09897000000001 + ], + [ + -89.91444, + 64.03273 + ], + [ + -90.70398, + 63.610170000000004 + ], + [ + -90.77004000000001, + 62.960210000000004 + ], + [ + -91.93342, + 62.83508 + ], + [ + -93.15698, + 62.02469000000001 + ], + [ + -94.24153, + 60.89865 + ], + [ + -94.62930999999999, + 60.11021 + ], + [ + -94.6846, + 58.94882 + ], + [ + -93.21502000000001, + 58.78212 + ], + [ + -92.76462000000001, + 57.84571 + ], + [ + -92.29702999999999, + 57.08709 + ], + [ + -90.89769, + 57.28468 + ], + [ + -89.03953, + 56.85172 + ], + [ + -88.03978000000001, + 56.47162 + ], + [ + -87.32421, + 55.999140000000004 + ], + [ + -86.07121, + 55.72383 + ], + [ + -85.01181000000001, + 55.302600000000005 + ], + [ + -83.36055, + 55.24489 + ], + [ + -82.27285, + 55.14832 + ], + [ + -82.43620000000001, + 54.282270000000004 + ], + [ + -82.12502, + 53.27703 + ], + [ + -81.40075, + 52.157880000000006 + ], + [ + -79.91289, + 51.208420000000004 + ], + [ + -79.14301, + 51.533930000000005 + ], + [ + -78.60191, + 52.56208 + ], + [ + -79.12421, + 54.14145 + ], + [ + -79.82958, + 54.66772 + ], + [ + -78.22874, + 55.136449999999996 + ], + [ + -77.0956, + 55.83741 + ], + [ + -76.54137, + 56.53423000000001 + ], + [ + -76.62319000000001, + 57.20263 + ], + [ + -77.30226, + 58.05209 + ], + [ + -78.51688, + 58.80458 + ], + [ + -77.33676, + 59.852610000000006 + ], + [ + -77.77272, + 60.75788000000001 + ], + [ + -78.10687, + 62.31964000000001 + ], + [ + -77.41067, + 62.55053 + ], + [ + -75.69621000000001, + 62.2784 + ], + [ + -74.6682, + 62.181110000000004 + ], + [ + -73.83988000000001, + 62.4438 + ], + [ + -72.90853, + 62.10507 + ], + [ + -71.67708, + 61.52535 + ], + [ + -71.37369000000001, + 61.137170000000005 + ], + [ + -69.59042, + 61.06141 + ], + [ + -69.62033, + 60.221250000000005 + ], + [ + -69.28790000000001, + 58.95736 + ], + [ + -68.37455, + 58.80106 + ], + [ + -67.64976, + 58.21206 + ], + [ + -66.20178, + 58.76731 + ], + [ + -65.24517, + 59.87071 + ], + [ + -64.58352000000001, + 60.33558 + ], + [ + -63.804750000000006, + 59.442600000000006 + ], + [ + -62.502359999999996, + 58.16708 + ], + [ + -61.396550000000005, + 56.96745000000001 + ], + [ + -61.798660000000005, + 56.33945 + ], + [ + -60.46853, + 55.775479999999995 + ], + [ + -59.56962, + 55.20407 + ], + [ + -57.97508, + 54.94549000000001 + ], + [ + -57.3332, + 54.6265 + ], + [ + -56.93689, + 53.780319999999996 + ], + [ + -56.15811, + 53.647490000000005 + ], + [ + -55.75632, + 53.27036 + ], + [ + -55.68338, + 52.146640000000005 + ], + [ + -56.40916000000001, + 51.770700000000005 + ], + [ + -57.12691, + 51.419720000000005 + ], + [ + -58.77482, + 51.0643 + ], + [ + -60.03309000000001, + 50.24277 + ], + [ + -61.72366, + 50.08046 + ], + [ + -63.86251, + 50.29099 + ], + [ + -65.36331, + 50.2982 + ], + [ + -66.39905, + 50.228970000000004 + ], + [ + -67.23631, + 49.511559999999996 + ], + [ + -68.51114, + 49.068360000000006 + ], + [ + -69.95362, + 47.74488 + ], + [ + -71.10458, + 46.82171 + ], + [ + -70.25522, + 46.986059999999995 + ], + [ + -68.65, + 48.3 + ], + [ + -66.55243, + 49.1331 + ], + [ + -65.05626, + 49.232780000000005 + ], + [ + -64.17099, + 48.74248 + ], + [ + -65.11545000000001, + 48.07085 + ], + [ + -64.79854, + 46.99297 + ], + [ + -64.47219, + 46.238490000000006 + ], + [ + -63.17329000000001, + 45.73902 + ], + [ + -61.520720000000004, + 45.883770000000005 + ], + [ + -60.518150000000006, + 47.00793 + ], + [ + -60.448600000000006, + 46.28264 + ], + [ + -59.80287, + 45.9204 + ], + [ + -61.03988, + 45.265249999999995 + ], + [ + -63.254709999999996, + 44.67014 + ], + [ + -64.24656, + 44.265530000000005 + ], + [ + -65.36406000000001, + 43.54523 + ], + [ + -66.1234, + 43.61867 + ], + [ + -66.16173, + 44.46512 + ], + [ + -64.42549, + 45.29204 + ], + [ + -66.02605000000001, + 45.25931 + ], + [ + -67.13741, + 45.13753 + ], + [ + -67.79134, + 45.70281000000001 + ], + [ + -67.79046000000001, + 47.066359999999996 + ], + [ + -68.23444, + 47.354859999999974 + ], + [ + -68.90500000000003, + 47.18500000000006 + ], + [ + -69.237216, + 47.447781 + ], + [ + -69.99997, + 46.69307 + ], + [ + -70.305, + 45.915 + ], + [ + -70.66, + 45.46 + ], + [ + -71.08482000000004, + 45.30524000000014 + ], + [ + -71.405, + 45.254999999999995 + ], + [ + -71.50506, + 45.0082 + ], + [ + -73.34783, + 45.00738 + ], + [ + -74.86700000000002, + 45.000480000000096 + ], + [ + -75.31821000000001, + 44.81645 + ], + [ + -76.375, + 44.09631 + ], + [ + -76.50000000000001, + 44.01845889375865 + ], + [ + -76.82003414580558, + 43.628784288093755 + ], + [ + -77.7378850979577, + 43.62905558936328 + ], + [ + -78.72027991404235, + 43.62508942318493 + ], + [ + -79.17167355011186, + 43.46633942318426 + ], + [ + -79.01, + 43.27 + ], + [ + -78.92, + 42.964999999999996 + ], + [ + -78.93936214874375, + 42.86361135514798 + ], + [ + -80.24744767934794, + 42.36619985612255 + ], + [ + -81.27774654816716, + 42.209025987306816 + ], + [ + -82.4392777167916, + 41.675105088867326 + ], + [ + -82.69008928092023, + 41.675105088867326 + ], + [ + -83.029810146807, + 41.83279572200598 + ], + [ + -83.14199968131264, + 41.975681057292874 + ], + [ + -83.12, + 42.08 + ], + [ + -82.9, + 42.43 + ], + [ + -82.42999999999999, + 42.980000000000004 + ], + [ + -82.13764238150395, + 43.57108755143997 + ], + [ + -82.33776312543114, + 44.440000000000055 + ], + [ + -82.55092464875821, + 45.34751658790543 + ], + [ + -83.59285071484311, + 45.81689362241252 + ], + [ + -83.46955074739469, + 45.994686387712534 + ], + [ + -83.61613094759059, + 46.116926988299014 + ], + [ + -83.89076534700574, + 46.116926988299014 + ], + [ + -84.0918512641615, + 46.27541860613826 + ], + [ + -84.1421195136734, + 46.51222585711571 + ], + [ + -84.33670000000001, + 46.408770000000004 + ], + [ + -84.60490000000004, + 46.439599999999984 + ], + [ + -84.54374874544584, + 46.538684190449146 + ], + [ + -84.77923824739992, + 46.63710195574902 + ], + [ + -84.8760798815149, + 46.90008331968238 + ], + [ + -85.65236324740341, + 47.22021881773051 + ], + [ + -86.46199083122826, + 47.553338019392 + ], + [ + -87.43979262330028, + 47.94 + ], + [ + -88.37811418328671, + 48.302917588893706 + ], + [ + -89.27291744663665, + 48.01980825458281 + ], + [ + -89.60000000000002, + 48.010000000000105 + ], + [ + -90.83, + 48.27 + ], + [ + -91.64, + 48.14 + ], + [ + -92.61000000000001, + 48.44999999999993 + ], + [ + -93.63087000000002, + 48.609260000000006 + ], + [ + -94.32914000000001, + 48.67074 + ], + [ + -94.64, + 48.84 + ], + [ + -94.81758000000002, + 49.38905 + ], + [ + -95.15609, + 49.38425000000001 + ], + [ + -95.15906950917206, + 49 + ], + [ + -97.2287200000048, + 49.0007 + ], + [ + -100.65000000000003, + 49.000000000000114 + ], + [ + -104.04826000000003, + 48.99986000000007 + ], + [ + -107.05000000000001, + 49 + ], + [ + -110.05000000000001, + 49 + ], + [ + -113, + 49 + ], + [ + -116.04818, + 49 + ], + [ + -117.03121, + 49 + ], + [ + -120, + 49.000000000000114 + ], + [ + -122.84000000000003, + 49.000000000000114 + ] + ] + ], + [ + [ + [ + -83.99367000000001, + 62.452799999999996 + ], + [ + -83.25048, + 62.91409 + ], + [ + -81.87699, + 62.90458 + ], + [ + -81.89825, + 62.7108 + ], + [ + -83.06857000000001, + 62.159220000000005 + ], + [ + -83.77462000000001, + 62.18231 + ], + [ + -83.99367000000001, + 62.452799999999996 + ] + ] + ], + [ + [ + [ + -79.77583312988281, + 72.8029022216797 + ], + [ + -80.87609863281251, + 73.33318328857422 + ], + [ + -80.83388519287111, + 73.69318389892578 + ], + [ + -80.35305786132812, + 73.75971984863281 + ], + [ + -78.06443786621094, + 73.65193176269531 + ], + [ + -76.34, + 73.10268498995305 + ], + [ + -76.25140380859375, + 72.82638549804688 + ], + [ + -77.31443786621094, + 72.85554504394531 + ], + [ + -78.39167022705078, + 72.87665557861328 + ], + [ + -79.4862518310547, + 72.74220275878906 + ], + [ + -79.77583312988281, + 72.8029022216797 + ] + ] + ], + [ + [ + [ + -80.315395, + 62.08556500000001 + ], + [ + -79.92939, + 62.3856 + ], + [ + -79.52002, + 62.363710000000005 + ], + [ + -79.26582, + 62.158674999999995 + ], + [ + -79.65752, + 61.63308 + ], + [ + -80.09956000000001, + 61.71810000000001 + ], + [ + -80.36215, + 62.016490000000005 + ], + [ + -80.315395, + 62.08556500000001 + ] + ] + ], + [ + [ + [ + -93.61275590694046, + 74.97999726022438 + ], + [ + -94.15690873897391, + 74.59234650338688 + ], + [ + -95.60868058956564, + 74.66686391875176 + ], + [ + -96.82093217648455, + 74.92762319609658 + ], + [ + -96.28858740922982, + 75.37782827422338 + ], + [ + -94.85081987178917, + 75.64721751576089 + ], + [ + -93.97774654821797, + 75.29648956979595 + ], + [ + -93.61275590694046, + 74.97999726022438 + ] + ] + ], + [ + [ + [ + -93.84000301794399, + 77.51999726023455 + ], + [ + -94.29560828324529, + 77.49134267852868 + ], + [ + -96.16965410031007, + 77.55511139597685 + ], + [ + -96.43630449093614, + 77.83462921824362 + ], + [ + -94.42257727738641, + 77.820004787905 + ], + [ + -93.7206562975659, + 77.63433136668031 + ], + [ + -93.84000301794399, + 77.51999726023455 + ] + ] + ], + [ + [ + [ + -96.75439876990876, + 78.76581268992702 + ], + [ + -95.5592779202946, + 78.41831452098033 + ], + [ + -95.83029496944934, + 78.05694122996324 + ], + [ + -97.30984290239799, + 77.85059723582181 + ], + [ + -98.12428931353404, + 78.08285696075761 + ], + [ + -98.55286780474668, + 78.45810537384507 + ], + [ + -98.63198442258553, + 78.87193024363837 + ], + [ + -97.33723141151266, + 78.83198436147676 + ], + [ + -96.75439876990876, + 78.76581268992702 + ] + ] + ], + [ + [ + [ + -88.15035030796028, + 74.39230703398503 + ], + [ + -89.7647220527584, + 74.51555532500116 + ], + [ + -92.42244096552946, + 74.83775788034099 + ], + [ + -92.76828548864282, + 75.38681997344214 + ], + [ + -92.88990597204175, + 75.88265534128267 + ], + [ + -93.89382402217599, + 76.31924367950056 + ], + [ + -95.9624574450358, + 76.4413809272224 + ], + [ + -97.1213789538295, + 76.7510777859476 + ], + [ + -96.74512285031237, + 77.16138865834507 + ], + [ + -94.68408586299944, + 77.09787832305837 + ], + [ + -93.57392106807313, + 76.77629588490605 + ], + [ + -91.6050231595366, + 76.7785179714946 + ], + [ + -90.7418458727493, + 76.44959747995681 + ], + [ + -90.96966142450802, + 76.07401317005947 + ], + [ + -89.82223792189926, + 75.84777374948565 + ], + [ + -89.18708289259985, + 75.61016551380762 + ], + [ + -87.83827633334965, + 75.56618886992725 + ], + [ + -86.37919226758864, + 75.4824213731821 + ], + [ + -84.78962521029058, + 75.69920400664653 + ], + [ + -82.75344458691006, + 75.78431509063124 + ], + [ + -81.12853084992436, + 75.71398346628199 + ], + [ + -80.05751095245915, + 75.33684886341591 + ], + [ + -79.83393286814837, + 74.92312734648716 + ], + [ + -80.45777075877587, + 74.65730377877777 + ], + [ + -81.94884253612557, + 74.44245901152432 + ], + [ + -83.22889360221143, + 74.56402781849094 + ], + [ + -86.09745235873332, + 74.41003205026117 + ], + [ + -88.15035030796028, + 74.39230703398503 + ] + ] + ], + [ + [ + [ + -111.26444332563088, + 78.15295604116154 + ], + [ + -109.85445187054711, + 77.99632477488488 + ], + [ + -110.18693803591302, + 77.69701487905034 + ], + [ + -112.0511911690585, + 77.4092288276169 + ], + [ + -113.53427893761912, + 77.73220652944111 + ], + [ + -112.7245867582539, + 78.05105011668196 + ], + [ + -111.26444332563088, + 78.15295604116154 + ] + ] + ], + [ + [ + [ + -110.96366065147602, + 78.8044408230652 + ], + [ + -109.6631457182026, + 78.60197256134565 + ], + [ + -110.88131425661892, + 78.40691986765997 + ], + [ + -112.54209143761516, + 78.4079017198735 + ], + [ + -112.52589087609164, + 78.55055451121522 + ], + [ + -111.5000103422334, + 78.8499935981305 + ], + [ + -110.96366065147602, + 78.8044408230652 + ] + ] + ], + [ + [ + [ + -55.600218268442056, + 51.31707469339794 + ], + [ + -56.13403581401709, + 50.68700979267928 + ], + [ + -56.795881720595276, + 49.81230866149089 + ], + [ + -56.14310502788433, + 50.15011749938286 + ], + [ + -55.471492275603, + 49.93581533466846 + ], + [ + -55.82240108908096, + 49.58712860777905 + ], + [ + -54.935142584845636, + 49.3130109726868 + ], + [ + -54.473775397343786, + 49.556691189159125 + ], + [ + -53.47654944519137, + 49.24913890237404 + ], + [ + -53.786013759971254, + 48.516780503933624 + ], + [ + -53.08613399922626, + 48.68780365660358 + ], + [ + -52.958648240762216, + 48.15716421161447 + ], + [ + -52.64809872090421, + 47.53554840757552 + ], + [ + -53.069158291218386, + 46.65549876564492 + ], + [ + -53.521456264853, + 46.61829173439477 + ], + [ + -54.17893551290251, + 46.80706574155698 + ], + [ + -53.9618686590605, + 47.62520701760193 + ], + [ + -54.24048214376214, + 47.752279364607645 + ], + [ + -55.40077307801157, + 46.884993801453135 + ], + [ + -55.99748084168583, + 46.919720363953275 + ], + [ + -55.29121904155279, + 47.38956248635099 + ], + [ + -56.250798712780586, + 47.632545070987376 + ], + [ + -57.32522925477708, + 47.57280711525797 + ], + [ + -59.26601518414682, + 47.60334788674247 + ], + [ + -59.419494188053676, + 47.899453843774886 + ], + [ + -58.79658647320744, + 48.25152537697942 + ], + [ + -59.23162451845657, + 48.52318838153781 + ], + [ + -58.3918049790652, + 49.12558055276418 + ], + [ + -57.35868974468606, + 50.71827403421587 + ], + [ + -56.738650071832026, + 51.28743825947855 + ], + [ + -55.87097693543532, + 51.63209422464921 + ], + [ + -55.40697424988659, + 51.5882726100657 + ], + [ + -55.600218268442056, + 51.31707469339794 + ] + ] + ], + [ + [ + [ + -83.88262630891977, + 65.10961782496354 + ], + [ + -82.78757687043883, + 64.76669302027467 + ], + [ + -81.6420137193926, + 64.45513580998697 + ], + [ + -81.55344031444432, + 63.97960928003714 + ], + [ + -80.81736121287886, + 64.057485663501 + ], + [ + -80.10345130076664, + 63.72598135034862 + ], + [ + -80.99101986359572, + 63.41124603947496 + ], + [ + -82.54717810741704, + 63.65172231714521 + ], + [ + -83.10879757356511, + 64.10187571883971 + ], + [ + -84.10041663281388, + 63.569711819098 + ], + [ + -85.52340471061905, + 63.052379055424055 + ], + [ + -85.8667687649824, + 63.63725291610349 + ], + [ + -87.22198320183678, + 63.54123810490519 + ], + [ + -86.35275977247133, + 64.0358332383707 + ], + [ + -86.2248864407651, + 64.82291697860823 + ], + [ + -85.88384782585486, + 65.7387783881171 + ], + [ + -85.1613079495499, + 65.6572846543928 + ], + [ + -84.97576371940592, + 65.21751821558898 + ], + [ + -84.4640120104195, + 65.37177236598022 + ], + [ + -83.88262630891977, + 65.10961782496354 + ] + ] + ], + [ + [ + [ + -78.77063859731078, + 72.35217316353418 + ], + [ + -77.8246239895596, + 72.74961660429098 + ], + [ + -75.60584469267573, + 72.2436784939374 + ], + [ + -74.228616095665, + 71.76714427355789 + ], + [ + -74.09914079455771, + 71.33084015571758 + ], + [ + -72.24222571479768, + 71.55692454699452 + ], + [ + -71.20001542833518, + 70.92001251899718 + ], + [ + -68.7860542466849, + 70.52502370877427 + ], + [ + -67.91497046575694, + 70.12194753689765 + ], + [ + -66.9690333726542, + 69.18608734809182 + ], + [ + -68.8051228502006, + 68.72019847276444 + ], + [ + -66.4498660956339, + 68.06716339789203 + ], + [ + -64.86231441919524, + 67.84753856065159 + ], + [ + -63.424934454996794, + 66.92847321234059 + ], + [ + -61.851981370680605, + 66.86212067327783 + ], + [ + -62.16317684594226, + 66.16025136988962 + ], + [ + -63.918444383384184, + 64.9986685248329 + ], + [ + -65.14886023625368, + 65.42603261988667 + ], + [ + -66.72121904159852, + 66.38804108343219 + ], + [ + -68.015016038674, + 66.26272573512439 + ], + [ + -68.1412874009792, + 65.68978913030439 + ], + [ + -67.08964616562342, + 65.10845510523696 + ], + [ + -65.73208045109976, + 64.64840566675856 + ], + [ + -65.32016760930125, + 64.38273712834605 + ], + [ + -64.66940629744968, + 63.392926744227495 + ], + [ + -65.01380388045888, + 62.67418508569598 + ], + [ + -66.27504472519048, + 62.94509878198612 + ], + [ + -68.7831862046927, + 63.74567007105183 + ], + [ + -67.36968075221309, + 62.88396556258484 + ], + [ + -66.32829728866726, + 62.28007477482201 + ], + [ + -66.16556820338015, + 61.93089712182582 + ], + [ + -68.87736650254465, + 62.330149237712824 + ], + [ + -71.02343705919385, + 62.91070811629588 + ], + [ + -72.23537858751902, + 63.39783600529522 + ], + [ + -71.88627844917127, + 63.67998932560887 + ], + [ + -73.37830624051838, + 64.19396312118384 + ], + [ + -74.83441891142263, + 64.6790756293238 + ], + [ + -74.81850257027673, + 64.38909332951793 + ], + [ + -77.70997982452008, + 64.22954234481678 + ], + [ + -78.5559488593542, + 64.57290639918013 + ], + [ + -77.89728105336198, + 65.30919220647475 + ], + [ + -76.01827429879717, + 65.32696889918314 + ], + [ + -73.95979529488268, + 65.45476471624094 + ], + [ + -74.29388342964964, + 65.81177134872938 + ], + [ + -73.94491248238262, + 66.31057811142666 + ], + [ + -72.65116716173942, + 67.28457550726391 + ], + [ + -72.92605994331605, + 67.72692576768235 + ], + [ + -73.31161780464572, + 68.06943716091287 + ], + [ + -74.84330725777684, + 68.55462718370127 + ], + [ + -76.86910091826672, + 68.89473562283025 + ], + [ + -76.22864905465738, + 69.14776927354741 + ], + [ + -77.28736996123715, + 69.76954010688321 + ], + [ + -78.1686339993266, + 69.82648753526887 + ], + [ + -78.95724219431673, + 70.16688019477543 + ], + [ + -79.49245500356366, + 69.87180776638884 + ], + [ + -81.30547095409176, + 69.74318512641436 + ], + [ + -84.94470618359851, + 69.96663401964442 + ], + [ + -87.06000342481789, + 70.26000112576538 + ], + [ + -88.68171322300148, + 70.4107412787608 + ], + [ + -89.51341956252303, + 70.76203766548095 + ], + [ + -88.46772111688082, + 71.21818553332132 + ], + [ + -89.88815121128755, + 71.22255219184997 + ], + [ + -90.20516028518205, + 72.23507436796079 + ], + [ + -89.436576707705, + 73.12946421985238 + ], + [ + -88.40824154331287, + 73.53788890247121 + ], + [ + -85.82615108920098, + 73.80381582304518 + ], + [ + -86.56217851433412, + 73.15744700793844 + ], + [ + -85.77437130404454, + 72.53412588163387 + ], + [ + -84.85011247428822, + 73.34027822538708 + ], + [ + -82.31559017610101, + 73.7509508328106 + ], + [ + -80.60008765330768, + 72.71654368762417 + ], + [ + -80.74894161652443, + 72.06190664335072 + ], + [ + -78.77063859731078, + 72.35217316353418 + ] + ] + ], + [ + [ + [ + -94.50365759965237, + 74.13490672473922 + ], + [ + -92.42001217321173, + 74.1000251329422 + ], + [ + -90.50979285354263, + 73.85673248971206 + ], + [ + -92.00396521682987, + 72.96624420845852 + ], + [ + -93.19629553910026, + 72.77199249947334 + ], + [ + -94.26904659704726, + 72.02459625923599 + ], + [ + -95.40985551632266, + 72.06188080513458 + ], + [ + -96.03374508338244, + 72.94027680123183 + ], + [ + -96.01826799191102, + 73.43742991809582 + ], + [ + -95.49579342322404, + 73.86241689726417 + ], + [ + -94.50365759965237, + 74.13490672473922 + ] + ] + ], + [ + [ + [ + -122.85492448615902, + 76.11654287383568 + ], + [ + -122.85492529360326, + 76.11654287383568 + ], + [ + -121.15753536032824, + 76.86450755482828 + ], + [ + -119.1039389718211, + 77.51221995717462 + ], + [ + -117.570130784966, + 77.4983189968881 + ], + [ + -116.19858659550738, + 77.6452867703262 + ], + [ + -116.33581336145845, + 76.87696157501061 + ], + [ + -117.10605058476882, + 76.53003184681911 + ], + [ + -118.04041215703819, + 76.48117178008714 + ], + [ + -119.89931758688572, + 76.053213406062 + ], + [ + -121.49999507712648, + 75.90001862253276 + ], + [ + -122.85492448615902, + 76.11654287383568 + ] + ] + ], + [ + [ + [ + -132.71000788443126, + 54.04000931542356 + ], + [ + -131.74998958400334, + 54.12000438090922 + ], + [ + -132.049480347351, + 52.98462148702447 + ], + [ + -131.1790425218266, + 52.180432847698285 + ], + [ + -131.57782954982298, + 52.18237071390928 + ], + [ + -132.18042842677852, + 52.639707139692405 + ], + [ + -132.54999243231384, + 53.100014960332146 + ], + [ + -133.05461117875552, + 53.411468817755406 + ], + [ + -133.2396644827927, + 53.851080227262344 + ], + [ + -133.1800040417117, + 54.169975490935315 + ], + [ + -132.71000788443126, + 54.04000931542356 + ] + ] + ], + [ + [ + [ + -105.4922891914932, + 79.30159393992916 + ], + [ + -103.52928239623795, + 79.16534902619163 + ], + [ + -100.8251580472688, + 78.80046173777872 + ], + [ + -100.0601918200522, + 78.32475434031589 + ], + [ + -99.67093909381364, + 77.90754466420744 + ], + [ + -101.30394019245301, + 78.01898489044486 + ], + [ + -102.94980872273302, + 78.34322866486023 + ], + [ + -105.17613277873151, + 78.3803323432458 + ], + [ + -104.21042945027713, + 78.67742015249176 + ], + [ + -105.41958045125853, + 78.91833567983649 + ], + [ + -105.4922891914932, + 79.30159393992916 + ] + ] + ], + [ + [ + [ + -123.51000158755119, + 48.51001089130341 + ], + [ + -124.01289078839955, + 48.37084625914139 + ], + [ + -125.65501277733838, + 48.8250045843385 + ], + [ + -125.95499446679275, + 49.17999583596759 + ], + [ + -126.85000443587185, + 49.53000031188043 + ], + [ + -127.02999344954443, + 49.81499583597008 + ], + [ + -128.0593363043662, + 49.9949590114266 + ], + [ + -128.44458410710214, + 50.539137681676095 + ], + [ + -128.35841365625546, + 50.77064809834371 + ], + [ + -127.30858109602994, + 50.552573554071955 + ], + [ + -126.69500097721235, + 50.400903225295394 + ], + [ + -125.7550066738232, + 50.29501821552935 + ], + [ + -125.4150015875588, + 49.95000051533259 + ], + [ + -124.92076818911934, + 49.475274970083376 + ], + [ + -123.92250870832106, + 49.06248362893581 + ], + [ + -123.51000158755119, + 48.51001089130341 + ] + ] + ], + [ + [ + [ + -121.53787999999997, + 74.44893000000002 + ], + [ + -120.10978, + 74.24135000000001 + ], + [ + -117.55563999999993, + 74.18576999999993 + ], + [ + -116.58442000000002, + 73.89607000000007 + ], + [ + -115.51080999999999, + 73.47519 + ], + [ + -116.76793999999995, + 73.22291999999999 + ], + [ + -119.22000000000003, + 72.51999999999998 + ], + [ + -120.45999999999998, + 71.82000000000005 + ], + [ + -120.45999999999998, + 71.38360179308756 + ], + [ + -123.09218999999996, + 70.90164000000004 + ], + [ + -123.62, + 71.34000000000009 + ], + [ + -125.92894873747338, + 71.86868846301138 + ], + [ + -125.49999999999994, + 72.29226081179502 + ], + [ + -124.80729000000002, + 73.02255999999994 + ], + [ + -123.93999999999994, + 73.68000000000012 + ], + [ + -124.91774999999996, + 74.29275000000013 + ], + [ + -121.53787999999997, + 74.44893000000002 + ] + ] + ], + [ + [ + [ + -107.81943000000001, + 75.84552000000001 + ], + [ + -106.92893000000001, + 76.01282 + ], + [ + -105.881, + 75.96940000000001 + ], + [ + -105.70498, + 75.47951 + ], + [ + -106.31347000000001, + 75.00527 + ], + [ + -109.70000000000002, + 74.85000000000001 + ], + [ + -112.22306999999999, + 74.41696 + ], + [ + -113.74381, + 74.39427 + ], + [ + -113.87135, + 74.72029 + ], + [ + -111.79420999999999, + 75.16250000000001 + ], + [ + -116.31221, + 75.04343 + ], + [ + -117.7104, + 75.2222 + ], + [ + -116.34602000000001, + 76.19903000000001 + ], + [ + -115.40487, + 76.47887 + ], + [ + -112.59056000000001, + 76.14134 + ], + [ + -110.81422, + 75.54919 + ], + [ + -109.06710000000001, + 75.47321000000001 + ], + [ + -110.49726000000001, + 76.42982 + ], + [ + -109.58109999999999, + 76.79417 + ], + [ + -108.54858999999999, + 76.67832000000001 + ], + [ + -108.21141, + 76.20168000000001 + ], + [ + -107.81943000000001, + 75.84552000000001 + ] + ] + ], + [ + [ + [ + -106.52258999999992, + 73.07601 + ], + [ + -105.40245999999996, + 72.67259000000007 + ], + [ + -104.77484000000004, + 71.6984000000001 + ], + [ + -104.4647599999999, + 70.99297000000007 + ], + [ + -102.78537, + 70.49776000000003 + ], + [ + -100.98077999999992, + 70.02431999999999 + ], + [ + -101.08928999999995, + 69.58447000000012 + ], + [ + -102.73115999999993, + 69.50402000000003 + ], + [ + -102.09329000000002, + 69.11962000000011 + ], + [ + -102.43024000000003, + 68.75281999999999 + ], + [ + -104.24000000000001, + 68.91000000000008 + ], + [ + -105.96000000000004, + 69.18000000000012 + ], + [ + -107.12254000000001, + 69.11922000000004 + ], + [ + -108.99999999999994, + 68.78000000000003 + ], + [ + -111.53414887520017, + 68.63005915681794 + ], + [ + -113.31320000000005, + 68.53553999999997 + ], + [ + -113.85495999999989, + 69.00744000000009 + ], + [ + -115.22000000000003, + 69.28000000000009 + ], + [ + -116.10793999999999, + 69.16821000000004 + ], + [ + -117.34000000000003, + 69.9600000000001 + ], + [ + -116.67472999999995, + 70.06655 + ], + [ + -115.13112000000001, + 70.23730000000006 + ], + [ + -113.72140999999999, + 70.1923700000001 + ], + [ + -112.41610000000003, + 70.36637999999999 + ], + [ + -114.35000000000002, + 70.60000000000002 + ], + [ + -116.48684000000003, + 70.52044999999998 + ], + [ + -117.90480000000002, + 70.54056000000014 + ], + [ + -118.43238000000002, + 70.90920000000006 + ], + [ + -116.11311, + 71.30917999999997 + ], + [ + -117.65567999999996, + 71.29520000000002 + ], + [ + -119.40199000000001, + 71.55858999999998 + ], + [ + -118.56266999999997, + 72.30785000000003 + ], + [ + -117.86641999999995, + 72.70594000000006 + ], + [ + -115.18909000000002, + 73.31459000000012 + ], + [ + -114.16716999999994, + 73.1214500000001 + ], + [ + -114.66633999999999, + 72.65277000000009 + ], + [ + -112.44101999999992, + 72.95540000000011 + ], + [ + -111.05039, + 72.45040000000006 + ], + [ + -109.92034999999993, + 72.96113000000008 + ], + [ + -109.00653999999997, + 72.63335000000001 + ], + [ + -108.18834999999996, + 71.65089 + ], + [ + -107.68599, + 72.0654800000001 + ], + [ + -108.39639, + 73.08953000000008 + ], + [ + -107.51645000000002, + 73.23597999999998 + ], + [ + -106.52258999999992, + 73.07601 + ] + ] + ], + [ + [ + [ + -100.43836, + 72.70588000000001 + ], + [ + -101.54, + 73.36 + ], + [ + -100.35642000000001, + 73.84389 + ], + [ + -99.16387, + 73.63339 + ], + [ + -97.38, + 73.76 + ], + [ + -97.12, + 73.47 + ], + [ + -98.05359, + 72.99052 + ], + [ + -96.54, + 72.56 + ], + [ + -96.72000000000001, + 71.66 + ], + [ + -98.35966, + 71.27284999999999 + ], + [ + -99.32286, + 71.35639 + ], + [ + -100.01482, + 71.73827 + ], + [ + -102.5, + 72.51 + ], + [ + -102.48000000000002, + 72.83000000000001 + ], + [ + -100.43836, + 72.70588000000001 + ] + ] + ], + [ + [ + [ + -106.6, + 73.60000000000001 + ], + [ + -105.26, + 73.64 + ], + [ + -104.5, + 73.42 + ], + [ + -105.38000000000001, + 72.76 + ], + [ + -106.94, + 73.46000000000001 + ], + [ + -106.6, + 73.60000000000001 + ] + ] + ], + [ + [ + [ + -98.50000000000001, + 76.72 + ], + [ + -97.735585, + 76.25656000000001 + ], + [ + -97.70441500000001, + 75.74344 + ], + [ + -98.16000000000001, + 75 + ], + [ + -99.80874, + 74.89744 + ], + [ + -100.88365999999999, + 75.05736 + ], + [ + -100.86292000000002, + 75.64075 + ], + [ + -102.50209, + 75.5638 + ], + [ + -102.56552, + 76.3366 + ], + [ + -101.48973, + 76.30537 + ], + [ + -99.98349, + 76.64634 + ], + [ + -98.57699, + 76.58859 + ], + [ + -98.50000000000001, + 76.72 + ] + ] + ], + [ + [ + [ + -96.01644, + 80.60233000000001 + ], + [ + -95.32345000000001, + 80.90729 + ], + [ + -94.29843, + 80.97727 + ], + [ + -94.73542, + 81.20646000000002 + ], + [ + -92.40983999999999, + 81.25739000000003 + ], + [ + -91.13288999999999, + 80.72345000000003 + ], + [ + -89.45000000000002, + 80.50932203389831 + ], + [ + -87.81, + 80.32000000000001 + ], + [ + -87.02000000000001, + 79.66000000000001 + ], + [ + -85.81435, + 79.3369 + ], + [ + -87.18755999999999, + 79.0393 + ], + [ + -89.03535000000001, + 78.28723 + ], + [ + -90.80436, + 78.21533000000001 + ], + [ + -92.87669000000001, + 78.34333000000001 + ], + [ + -93.95116000000002, + 78.75099 + ], + [ + -93.93574, + 79.11373 + ], + [ + -93.14524, + 79.3801 + ], + [ + -94.974, + 79.37248 + ], + [ + -96.07614000000001, + 79.70502 + ], + [ + -96.70972, + 80.15777 + ], + [ + -96.01644, + 80.60233000000001 + ] + ] + ], + [ + [ + [ + -91.58702000000001, + 81.89429000000001 + ], + [ + -90.10000000000001, + 82.08500000000004 + ], + [ + -88.93227, + 82.11751000000001 + ], + [ + -86.97024, + 82.27961 + ], + [ + -85.5, + 82.65227345805702 + ], + [ + -84.260005, + 82.60000000000001 + ], + [ + -83.18, + 82.32 + ], + [ + -82.42, + 82.86000000000001 + ], + [ + -81.1, + 83.02 + ], + [ + -79.30664, + 83.13056 + ], + [ + -76.25, + 83.17205882352941 + ], + [ + -75.71878000000001, + 83.06404000000002 + ], + [ + -72.83153, + 83.23324000000001 + ], + [ + -70.66576500000001, + 83.16978075838284 + ], + [ + -68.50000000000001, + 83.10632151676572 + ], + [ + -65.82735, + 83.02801000000001 + ], + [ + -63.68, + 82.9 + ], + [ + -61.85, + 82.62860000000002 + ], + [ + -61.89388, + 82.36165000000001 + ], + [ + -64.334, + 81.92775000000002 + ], + [ + -66.75342, + 81.72527000000001 + ], + [ + -67.65755, + 81.50141 + ], + [ + -65.48031, + 81.50657000000002 + ], + [ + -67.84, + 80.90000000000003 + ], + [ + -69.4697, + 80.61683000000001 + ], + [ + -71.18, + 79.8 + ], + [ + -73.2428, + 79.63415 + ], + [ + -73.88000000000001, + 79.43016220480206 + ], + [ + -76.90773, + 79.32309000000001 + ], + [ + -75.52924, + 79.19766000000001 + ], + [ + -76.22046, + 79.01907 + ], + [ + -75.39345, + 78.52581 + ], + [ + -76.34354, + 78.18296000000001 + ], + [ + -77.88851000000001, + 77.89991 + ], + [ + -78.36269, + 77.50859000000001 + ], + [ + -79.75951, + 77.20967999999999 + ], + [ + -79.61965000000001, + 76.98336 + ], + [ + -77.91089000000001, + 77.022045 + ], + [ + -77.88911, + 76.777955 + ], + [ + -80.56125, + 76.17812 + ], + [ + -83.17439, + 76.45403 + ], + [ + -86.11184, + 76.29901000000001 + ], + [ + -87.60000000000001, + 76.42 + ], + [ + -89.49068, + 76.47239 + ], + [ + -89.6161, + 76.95213000000001 + ], + [ + -87.76739, + 77.17833 + ], + [ + -88.26, + 77.9 + ], + [ + -87.65, + 77.97022222222223 + ], + [ + -84.97634, + 77.53873 + ], + [ + -86.34, + 78.18 + ], + [ + -87.96191999999999, + 78.37181 + ], + [ + -87.15198000000001, + 78.75867 + ], + [ + -85.37868, + 78.99690000000001 + ], + [ + -85.09495, + 79.34543000000001 + ], + [ + -86.50734, + 79.73624 + ], + [ + -86.93179, + 80.25145 + ], + [ + -84.19844, + 80.20836 + ], + [ + -83.40869565217389, + 80.10000000000001 + ], + [ + -81.84823, + 80.46442 + ], + [ + -84.1, + 80.58 + ], + [ + -87.59895, + 80.51627 + ], + [ + -89.36663, + 80.85569000000001 + ], + [ + -90.2, + 81.26 + ], + [ + -91.36786000000001, + 81.5531 + ], + [ + -91.58702000000001, + 81.89429000000001 + ] + ] + ], + [ + [ + [ + -75.21597, + 67.44425 + ], + [ + -75.86588, + 67.14886 + ], + [ + -76.98687, + 67.09873 + ], + [ + -77.2364, + 67.58809000000001 + ], + [ + -76.81166, + 68.14856 + ], + [ + -75.89521, + 68.28721 + ], + [ + -75.11449999999999, + 68.01035999999999 + ], + [ + -75.10333, + 67.58202 + ], + [ + -75.21597, + 67.44425 + ] + ] + ], + [ + [ + [ + -96.25740120380055, + 69.49003035832177 + ], + [ + -95.64768120380054, + 69.10769035832178 + ], + [ + -96.26952120380055, + 68.75704035832177 + ], + [ + -97.61740120380055, + 69.06003035832177 + ], + [ + -98.43180120380055, + 68.95070035832177 + ], + [ + -99.79740120380055, + 69.40003035832177 + ], + [ + -98.91740120380055, + 69.71003035832177 + ], + [ + -98.21826120380055, + 70.14354035832177 + ], + [ + -97.15740120380055, + 69.86003035832177 + ], + [ + -96.55740120380055, + 69.68003035832177 + ], + [ + -96.25740120380055, + 69.49003035832177 + ] + ] + ], + [ + [ + [ + -64.51912, + 49.87304 + ], + [ + -64.17322, + 49.95718 + ], + [ + -62.858290000000004, + 49.70641 + ], + [ + -61.835584999999995, + 49.28855 + ], + [ + -61.806304999999995, + 49.10506000000001 + ], + [ + -62.29318, + 49.08717 + ], + [ + -63.589259999999996, + 49.400690000000004 + ], + [ + -64.51912, + 49.87304 + ] + ] + ], + [ + [ + [ + -64.01486, + 47.03601 + ], + [ + -63.6645, + 46.55001 + ], + [ + -62.9393, + 46.41587 + ], + [ + -62.012080000000005, + 46.44314 + ], + [ + -62.503910000000005, + 46.033390000000004 + ], + [ + -62.87433, + 45.968180000000004 + ], + [ + -64.14280000000001, + 46.39265 + ], + [ + -64.39261, + 46.72747 + ], + [ + -64.01486, + 47.03601 + ] + ] + ] + ] + }, + "properties": { + "continent": "North America", + "gdp_md_est": 1674000, + "iso_a3": "CAN", + "name": "Canada", + "pop_est": 35623680 + }, + "type": "Feature", + "bbox": [ + -140.99778, + 41.675105088867326, + -52.64809872090421, + 83.23324000000001 + ] + }, + { + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -122.84000000000003, + 49.000000000000114 + ], + [ + -120, + 49.000000000000114 + ], + [ + -117.03121, + 49 + ], + [ + -116.04818, + 49 + ], + [ + -113, + 49 + ], + [ + -110.05000000000001, + 49 + ], + [ + -107.05000000000001, + 49 + ], + [ + -104.04826000000003, + 48.99986000000007 + ], + [ + -100.65000000000003, + 49.000000000000114 + ], + [ + -97.2287200000048, + 49.0007 + ], + [ + -95.15906950917206, + 49 + ], + [ + -95.15609, + 49.38425000000001 + ], + [ + -94.81758000000002, + 49.38905 + ], + [ + -94.64, + 48.84 + ], + [ + -94.32914000000001, + 48.67074 + ], + [ + -93.63087000000002, + 48.609260000000006 + ], + [ + -92.61000000000001, + 48.44999999999993 + ], + [ + -91.64, + 48.14 + ], + [ + -90.83, + 48.27 + ], + [ + -89.60000000000002, + 48.010000000000105 + ], + [ + -89.27291744663665, + 48.01980825458281 + ], + [ + -88.37811418328671, + 48.302917588893706 + ], + [ + -87.43979262330028, + 47.94 + ], + [ + -86.46199083122826, + 47.553338019392 + ], + [ + -85.65236324740341, + 47.22021881773051 + ], + [ + -84.8760798815149, + 46.90008331968238 + ], + [ + -84.77923824739992, + 46.63710195574902 + ], + [ + -84.54374874544584, + 46.538684190449146 + ], + [ + -84.60490000000004, + 46.439599999999984 + ], + [ + -84.33670000000001, + 46.408770000000004 + ], + [ + -84.1421195136734, + 46.51222585711571 + ], + [ + -84.0918512641615, + 46.27541860613826 + ], + [ + -83.89076534700574, + 46.116926988299014 + ], + [ + -83.61613094759059, + 46.116926988299014 + ], + [ + -83.46955074739469, + 45.994686387712534 + ], + [ + -83.59285071484311, + 45.81689362241252 + ], + [ + -82.55092464875821, + 45.34751658790543 + ], + [ + -82.33776312543114, + 44.440000000000055 + ], + [ + -82.13764238150395, + 43.57108755143997 + ], + [ + -82.42999999999999, + 42.980000000000004 + ], + [ + -82.9, + 42.43 + ], + [ + -83.12, + 42.08 + ], + [ + -83.14199968131264, + 41.975681057292874 + ], + [ + -83.029810146807, + 41.83279572200598 + ], + [ + -82.69008928092023, + 41.675105088867326 + ], + [ + -82.4392777167916, + 41.675105088867326 + ], + [ + -81.27774654816716, + 42.209025987306816 + ], + [ + -80.24744767934794, + 42.36619985612255 + ], + [ + -78.93936214874375, + 42.86361135514798 + ], + [ + -78.92, + 42.964999999999996 + ], + [ + -79.01, + 43.27 + ], + [ + -79.17167355011186, + 43.46633942318426 + ], + [ + -78.72027991404235, + 43.62508942318493 + ], + [ + -77.7378850979577, + 43.62905558936328 + ], + [ + -76.82003414580558, + 43.628784288093755 + ], + [ + -76.50000000000001, + 44.01845889375865 + ], + [ + -76.375, + 44.09631 + ], + [ + -75.31821000000001, + 44.81645 + ], + [ + -74.86700000000002, + 45.000480000000096 + ], + [ + -73.34783, + 45.00738 + ], + [ + -71.50506, + 45.0082 + ], + [ + -71.405, + 45.254999999999995 + ], + [ + -71.08482000000004, + 45.30524000000014 + ], + [ + -70.66, + 45.46 + ], + [ + -70.305, + 45.915 + ], + [ + -69.99997, + 46.69307 + ], + [ + -69.237216, + 47.447781 + ], + [ + -68.90500000000003, + 47.18500000000006 + ], + [ + -68.23444, + 47.354859999999974 + ], + [ + -67.79046000000001, + 47.066359999999996 + ], + [ + -67.79134, + 45.70281000000001 + ], + [ + -67.13741, + 45.13753 + ], + [ + -66.96465999999998, + 44.809700000000134 + ], + [ + -68.03251999999998, + 44.325199999999995 + ], + [ + -69.05999999999995, + 43.980000000000075 + ], + [ + -70.11616999999995, + 43.68405000000013 + ], + [ + -70.64547563341102, + 43.09023834896402 + ], + [ + -70.81488999999999, + 42.865299999999934 + ], + [ + -70.82499999999999, + 42.33499999999998 + ], + [ + -70.49499999999995, + 41.80500000000001 + ], + [ + -70.07999999999998, + 41.78000000000003 + ], + [ + -70.185, + 42.145000000000095 + ], + [ + -69.88496999999995, + 41.92283000000009 + ], + [ + -69.96502999999996, + 41.63717000000014 + ], + [ + -70.63999999999999, + 41.47500000000002 + ], + [ + -71.12039000000004, + 41.49445000000014 + ], + [ + -71.8599999999999, + 41.32000000000005 + ], + [ + -72.29500000000002, + 41.26999999999998 + ], + [ + -72.87643000000003, + 41.220650000000035 + ], + [ + -73.71000000000004, + 40.93110235165449 + ], + [ + -72.24125999999995, + 41.119480000000124 + ], + [ + -71.94499999999988, + 40.930000000000064 + ], + [ + -73.34499999999997, + 40.63000000000005 + ], + [ + -73.98200000000003, + 40.62799999999993 + ], + [ + -73.95232499999997, + 40.75075000000004 + ], + [ + -74.25671, + 40.47351000000003 + ], + [ + -73.96243999999996, + 40.42763000000002 + ], + [ + -74.17838, + 39.70925999999997 + ], + [ + -74.90603999999996, + 38.93954000000002 + ], + [ + -74.98041, + 39.19640000000004 + ], + [ + -75.20002, + 39.248450000000105 + ], + [ + -75.52805000000001, + 39.49850000000009 + ], + [ + -75.32, + 38.960000000000036 + ], + [ + -75.07183476478986, + 38.782032230179254 + ], + [ + -75.05672999999996, + 38.40412000000009 + ], + [ + -75.37746999999996, + 38.015510000000006 + ], + [ + -75.94022999999999, + 37.21689000000009 + ], + [ + -76.03126999999995, + 37.25659999999999 + ], + [ + -75.72204999999985, + 37.93705000000011 + ], + [ + -76.23286999999999, + 38.319214999999986 + ], + [ + -76.35000000000002, + 39.14999999999998 + ], + [ + -76.54272499999996, + 38.71761500000008 + ], + [ + -76.32933000000003, + 38.08326000000005 + ], + [ + -76.98999793161352, + 38.23999176691336 + ], + [ + -76.30161999999996, + 37.91794499999992 + ], + [ + -76.25873999999999, + 36.96640000000008 + ], + [ + -75.97179999999997, + 36.89726000000002 + ], + [ + -75.8680399999999, + 36.55125000000004 + ], + [ + -75.72748999999999, + 35.55074000000013 + ], + [ + -76.36318, + 34.80854000000011 + ], + [ + -77.39763499999992, + 34.512009999999975 + ], + [ + -78.05496, + 33.92547000000002 + ], + [ + -78.55434999999989, + 33.86133000000012 + ], + [ + -79.06067000000002, + 33.493949999999984 + ], + [ + -79.20357000000001, + 33.158390000000054 + ], + [ + -80.30132499999996, + 32.509355000000085 + ], + [ + -80.86498, + 32.033300000000054 + ], + [ + -81.33629000000002, + 31.44049000000001 + ], + [ + -81.49041999999997, + 30.7299900000001 + ], + [ + -81.31371000000001, + 30.035520000000076 + ], + [ + -80.97999999999996, + 29.18000000000012 + ], + [ + -80.53558499999991, + 28.472129999999993 + ], + [ + -80.52999999999986, + 28.040000000000077 + ], + [ + -80.05653928497759, + 26.88000000000011 + ], + [ + -80.08801499999998, + 26.205764999999985 + ], + [ + -80.13155999999992, + 25.816775000000064 + ], + [ + -80.38103000000001, + 25.20616000000001 + ], + [ + -80.67999999999995, + 25.08000000000004 + ], + [ + -81.17212999999998, + 25.201260000000104 + ], + [ + -81.33000000000004, + 25.639999999999986 + ], + [ + -81.70999999999987, + 25.870000000000005 + ], + [ + -82.23999999999995, + 26.730000000000132 + ], + [ + -82.70515, + 27.495040000000074 + ], + [ + -82.85525999999999, + 27.886240000000043 + ], + [ + -82.64999999999998, + 28.550000000000125 + ], + [ + -82.92999999999995, + 29.10000000000008 + ], + [ + -83.70958999999999, + 29.936560000000043 + ], + [ + -84.09999999999997, + 30.09000000000009 + ], + [ + -85.10881999999998, + 29.636150000000043 + ], + [ + -85.28784000000002, + 29.68612000000013 + ], + [ + -85.7731, + 30.152610000000095 + ], + [ + -86.39999999999992, + 30.40000000000009 + ], + [ + -87.53035999999992, + 30.27433000000002 + ], + [ + -88.41781999999995, + 30.384900000000016 + ], + [ + -89.1804899999999, + 30.315980000000025 + ], + [ + -89.5938311784198, + 30.159994004836847 + ], + [ + -89.41373499999997, + 29.89418999999998 + ], + [ + -89.43, + 29.488639999999975 + ], + [ + -89.21767, + 29.291080000000022 + ], + [ + -89.40822999999995, + 29.159610000000043 + ], + [ + -89.77927999999997, + 29.307140000000118 + ], + [ + -90.15463, + 29.11743000000007 + ], + [ + -90.88022499999994, + 29.148535000000095 + ], + [ + -91.62678499999993, + 29.677000000000135 + ], + [ + -92.49905999999999, + 29.552300000000002 + ], + [ + -93.22636999999997, + 29.783750000000055 + ], + [ + -93.84841999999998, + 29.71363000000008 + ], + [ + -94.69, + 29.480000000000132 + ], + [ + -95.60025999999999, + 28.738630000000057 + ], + [ + -96.59403999999995, + 28.307480000000055 + ], + [ + -97.13999999999987, + 27.83000000000004 + ], + [ + -97.36999999999995, + 27.380000000000052 + ], + [ + -97.37999999999994, + 26.690000000000055 + ], + [ + -97.32999999999998, + 26.210000000000093 + ], + [ + -97.13999999999987, + 25.870000000000005 + ], + [ + -97.52999999999992, + 25.84000000000009 + ], + [ + -98.23999999999995, + 26.06000000000006 + ], + [ + -99.01999999999992, + 26.37000000000006 + ], + [ + -99.30000000000001, + 26.840000000000032 + ], + [ + -99.51999999999992, + 27.54000000000002 + ], + [ + -100.10999999999996, + 28.110000000000127 + ], + [ + -100.45584000000002, + 28.69612000000012 + ], + [ + -100.95759999999996, + 29.380710000000136 + ], + [ + -101.66239999999999, + 29.77930000000009 + ], + [ + -102.48000000000002, + 29.75999999999999 + ], + [ + -103.11000000000001, + 28.970000000000027 + ], + [ + -103.94, + 29.27000000000004 + ], + [ + -104.4569699999999, + 29.571960000000047 + ], + [ + -104.70574999999997, + 30.121730000000014 + ], + [ + -105.03737000000001, + 30.644019999999955 + ], + [ + -105.63159000000002, + 31.08383000000009 + ], + [ + -106.1429, + 31.399950000000047 + ], + [ + -106.50758999999988, + 31.754520000000014 + ], + [ + -108.24000000000001, + 31.754853718166373 + ], + [ + -108.24193999999994, + 31.342220000000054 + ], + [ + -109.03500000000003, + 31.341940000000136 + ], + [ + -111.02361000000002, + 31.334719999999948 + ], + [ + -113.30498, + 32.03914000000009 + ], + [ + -114.815, + 32.52528000000001 + ], + [ + -114.72138999999993, + 32.72082999999992 + ], + [ + -115.99134999999995, + 32.61239000000012 + ], + [ + -117.12775999999985, + 32.53533999999996 + ], + [ + -117.29593769127393, + 33.04622461520387 + ], + [ + -117.94400000000002, + 33.621236431201396 + ], + [ + -118.41060227589753, + 33.74090922312445 + ], + [ + -118.51989482279976, + 34.02778157757575 + ], + [ + -119.08100000000002, + 34.07799999999992 + ], + [ + -119.43884064201671, + 34.34847717828427 + ], + [ + -120.36777999999998, + 34.447110000000066 + ], + [ + -120.62286, + 34.60854999999998 + ], + [ + -120.74432999999999, + 35.15686000000011 + ], + [ + -121.71456999999992, + 36.161529999999914 + ], + [ + -122.54746999999998, + 37.551760000000115 + ], + [ + -122.51201000000003, + 37.78339000000011 + ], + [ + -122.95319, + 38.11371000000008 + ], + [ + -123.72720000000004, + 38.95166000000012 + ], + [ + -123.86516999999998, + 39.76699000000008 + ], + [ + -124.39807000000002, + 40.313199999999995 + ], + [ + -124.17885999999999, + 41.142020000000116 + ], + [ + -124.21370000000002, + 41.99964000000011 + ], + [ + -124.53283999999996, + 42.7659900000001 + ], + [ + -124.14213999999998, + 43.708380000000034 + ], + [ + -124.020535, + 44.615894999999966 + ], + [ + -123.89892999999995, + 45.52341000000007 + ], + [ + -124.079635, + 46.864750000000015 + ], + [ + -124.39566999999994, + 47.72017000000011 + ], + [ + -124.68721008300781, + 48.18443298339855 + ], + [ + -124.56610107421875, + 48.37971496582037 + ], + [ + -123.12, + 48.04000000000002 + ], + [ + -122.58735999999993, + 47.09600000000006 + ], + [ + -122.34000000000003, + 47.360000000000014 + ], + [ + -122.5, + 48.180000000000064 + ], + [ + -122.84000000000003, + 49.000000000000114 + ] + ] + ], + [ + [ + [ + -155.40214, + 20.07975 + ], + [ + -155.22452, + 19.99302 + ], + [ + -155.06226, + 19.8591 + ], + [ + -154.80741, + 19.50871 + ], + [ + -154.83147, + 19.453280000000003 + ], + [ + -155.22217, + 19.23972 + ], + [ + -155.54211, + 19.08348 + ], + [ + -155.68817, + 18.91619 + ], + [ + -155.93665, + 19.05939 + ], + [ + -155.90806, + 19.33888 + ], + [ + -156.07347000000001, + 19.70294 + ], + [ + -156.02368, + 19.81422 + ], + [ + -155.85008000000002, + 19.97729 + ], + [ + -155.91907, + 20.17395 + ], + [ + -155.86108000000002, + 20.267210000000002 + ], + [ + -155.78505, + 20.2487 + ], + [ + -155.40214, + 20.07975 + ] + ] + ], + [ + [ + [ + -155.99566000000002, + 20.76404 + ], + [ + -156.07926, + 20.643970000000003 + ], + [ + -156.41445, + 20.57241 + ], + [ + -156.58673, + 20.783 + ], + [ + -156.70167, + 20.8643 + ], + [ + -156.71054999999998, + 20.92676 + ], + [ + -156.61258, + 21.01249 + ], + [ + -156.25711, + 20.917450000000002 + ], + [ + -155.99566000000002, + 20.76404 + ] + ] + ], + [ + [ + [ + -156.75824, + 21.176840000000002 + ], + [ + -156.78933, + 21.068730000000002 + ], + [ + -157.32521, + 21.097770000000004 + ], + [ + -157.25027, + 21.219579999999997 + ], + [ + -156.75824, + 21.176840000000002 + ] + ] + ], + [ + [ + [ + -158.0252, + 21.71696 + ], + [ + -157.94161, + 21.65272 + ], + [ + -157.65283000000002, + 21.322170000000003 + ], + [ + -157.70703, + 21.26442 + ], + [ + -157.7786, + 21.27729 + ], + [ + -158.12667000000002, + 21.31244 + ], + [ + -158.2538, + 21.53919 + ], + [ + -158.29265, + 21.57912 + ], + [ + -158.0252, + 21.71696 + ] + ] + ], + [ + [ + [ + -159.36569, + 22.21494 + ], + [ + -159.34512, + 21.982000000000003 + ], + [ + -159.46372, + 21.88299 + ], + [ + -159.80051, + 22.065330000000003 + ], + [ + -159.74877, + 22.1382 + ], + [ + -159.5962, + 22.236179999999997 + ], + [ + -159.36569, + 22.21494 + ] + ] + ], + [ + [ + [ + -166.46779212142462, + 60.384169826897754 + ], + [ + -165.67442969466364, + 60.29360687930625 + ], + [ + -165.57916419173358, + 59.90998688418753 + ], + [ + -166.19277014876727, + 59.75444082298899 + ], + [ + -166.84833736882197, + 59.941406155020985 + ], + [ + -167.45527706609008, + 60.21306915957936 + ], + [ + -166.46779212142462, + 60.384169826897754 + ] + ] + ], + [ + [ + [ + -153.22872941792113, + 57.96896841087248 + ], + [ + -152.56479061583514, + 57.901427313866996 + ], + [ + -152.1411472239064, + 57.591058661522 + ], + [ + -153.00631405333692, + 57.11584219016593 + ], + [ + -154.0050902984581, + 56.734676825581076 + ], + [ + -154.51640275777004, + 56.99274892844669 + ], + [ + -154.67099280497118, + 57.46119578717253 + ], + [ + -153.7627795074415, + 57.81657461204373 + ], + [ + -153.22872941792113, + 57.96896841087248 + ] + ] + ], + [ + [ + [ + -140.98598761037601, + 69.71199839952635 + ], + [ + -140.986, + 69.712 + ], + [ + -140.9925, + 66.00003000000001 + ], + [ + -140.99778, + 60.30639000000001 + ], + [ + -140.013, + 60.27682000000001 + ], + [ + -139.03900000000002, + 60 + ], + [ + -138.34089, + 59.562110000000004 + ], + [ + -137.4525, + 58.905 + ], + [ + -136.47972000000004, + 59.46389000000005 + ], + [ + -135.47583, + 59.787780000000005 + ], + [ + -134.94500000000005, + 59.2705600000001 + ], + [ + -134.27111000000002, + 58.86111000000005 + ], + [ + -133.35556000000003, + 58.41028000000001 + ], + [ + -132.73042, + 57.692890000000006 + ], + [ + -131.70781, + 56.55212 + ], + [ + -130.00778000000003, + 55.915830000000085 + ], + [ + -129.98, + 55.285000000000004 + ], + [ + -130.53611, + 54.802780000000006 + ], + [ + -130.53610895273684, + 54.80275447679924 + ], + [ + -130.5361101894673, + 54.8027534043494 + ], + [ + -131.08581823797215, + 55.17890615500204 + ], + [ + -131.9672114671423, + 55.497775580459006 + ], + [ + -132.2500107428595, + 56.3699962428974 + ], + [ + -133.53918108435641, + 57.17888743756214 + ], + [ + -134.07806292029608, + 58.12306753196691 + ], + [ + -135.0382110322791, + 58.18771474876394 + ], + [ + -136.62806230995471, + 58.21220937767043 + ], + [ + -137.800006279686, + 58.49999542910376 + ], + [ + -139.867787041413, + 59.53776154238915 + ], + [ + -140.825273817133, + 59.727517401765056 + ], + [ + -142.57444353556446, + 60.08444651960497 + ], + [ + -143.9588809948799, + 59.999180406323376 + ], + [ + -145.92555681682788, + 60.45860972761426 + ], + [ + -147.11437394914665, + 60.884656073644635 + ], + [ + -148.22430620012761, + 60.67298940697714 + ], + [ + -148.01806555885082, + 59.97832896589364 + ], + [ + -148.57082251686086, + 59.914172675203304 + ], + [ + -149.72785783587585, + 59.70565827090553 + ], + [ + -150.60824337461642, + 59.368211168039466 + ], + [ + -151.7163927886833, + 59.15582103131993 + ], + [ + -151.85943315326722, + 59.744984035879554 + ], + [ + -151.40971900124717, + 60.72580272077937 + ], + [ + -150.3469414947325, + 61.03358755150987 + ], + [ + -150.62111080625704, + 61.2844249538544 + ], + [ + -151.89583919981683, + 60.727197984451266 + ], + [ + -152.57832984109558, + 60.061657212964235 + ], + [ + -154.01917212625764, + 59.35027944603428 + ], + [ + -153.28751135965317, + 58.86472768821977 + ], + [ + -154.23249243875847, + 58.14637360293051 + ], + [ + -155.3074914215102, + 57.727794501366304 + ], + [ + -156.30833472392305, + 57.422774359763594 + ], + [ + -156.55609737854638, + 56.97998484967064 + ], + [ + -158.11721655986779, + 56.46360809999419 + ], + [ + -158.43332129619714, + 55.99415355083852 + ], + [ + -159.60332739971741, + 55.56668610292013 + ], + [ + -160.28971961163427, + 55.643580634170576 + ], + [ + -161.22304765525777, + 55.364734605523495 + ], + [ + -162.23776607974105, + 55.02418691672011 + ], + [ + -163.06944658104638, + 54.68973704692712 + ], + [ + -164.78556922102717, + 54.40417308208214 + ], + [ + -164.94222632552007, + 54.57222483989534 + ], + [ + -163.84833960676565, + 55.03943146424609 + ], + [ + -162.87000139061595, + 55.34804311789321 + ], + [ + -161.80417497459607, + 55.89498647727038 + ], + [ + -160.5636047027812, + 56.00805451112501 + ], + [ + -160.07055986228448, + 56.41805532492873 + ], + [ + -158.6844429189195, + 57.01667511659787 + ], + [ + -158.46109737855403, + 57.21692129172885 + ], + [ + -157.72277035218391, + 57.57000051536306 + ], + [ + -157.55027442119362, + 58.328326321030204 + ], + [ + -157.04167497457698, + 58.91888458926172 + ], + [ + -158.19473120830554, + 58.61580231386978 + ], + [ + -158.51721798402303, + 58.78778148053732 + ], + [ + -159.0586061269288, + 58.42418610293163 + ], + [ + -159.71166704001737, + 58.93139028587632 + ], + [ + -159.98128882550017, + 58.572549140041644 + ], + [ + -160.3552711659965, + 59.07112335879361 + ], + [ + -161.3550034251151, + 58.670837714260756 + ], + [ + -161.96889360252632, + 58.67166453717738 + ], + [ + -162.05498653872465, + 59.26692536074745 + ], + [ + -161.8741707021354, + 59.63362132429057 + ], + [ + -162.51805904849212, + 59.98972361921386 + ], + [ + -163.8183414378202, + 59.79805573184336 + ], + [ + -164.66221757714652, + 60.26748444278263 + ], + [ + -165.3463877024748, + 60.50749563256238 + ], + [ + -165.3508318756519, + 61.073895168697504 + ], + [ + -166.12137915755602, + 61.50001902937623 + ], + [ + -165.73445187077058, + 62.074996853271784 + ], + [ + -164.9191786367179, + 62.63307648380794 + ], + [ + -164.56250790103934, + 63.14637848576302 + ], + [ + -163.75333248599708, + 63.21944896102377 + ], + [ + -163.06722449445786, + 63.05945872664802 + ], + [ + -162.26055538638175, + 63.54193573674115 + ], + [ + -161.53444983624863, + 63.455816962326764 + ], + [ + -160.7725066803211, + 63.766108100023246 + ], + [ + -160.9583351308426, + 64.22279857040274 + ], + [ + -161.51806840721218, + 64.40278758407527 + ], + [ + -160.77777767641481, + 64.78860382756642 + ], + [ + -161.39192623598765, + 64.77723501246231 + ], + [ + -162.4530500966689, + 64.55944468856819 + ], + [ + -162.75778601789415, + 64.33860545516876 + ], + [ + -163.54639421288428, + 64.5591604681905 + ], + [ + -164.96082984114514, + 64.44694509546883 + ], + [ + -166.42528825586447, + 64.68667206487066 + ], + [ + -166.8450042389391, + 65.08889557561452 + ], + [ + -168.11056006576715, + 65.66999705673675 + ], + [ + -166.70527116602193, + 66.08831777613938 + ], + [ + -164.47470964257548, + 66.5766600612975 + ], + [ + -163.65251176659564, + 66.5766600612975 + ], + [ + -163.78860165103623, + 66.07720734319668 + ], + [ + -161.67777442121013, + 66.11611969671242 + ], + [ + -162.48971452538004, + 66.73556509059512 + ], + [ + -163.71971696679117, + 67.11639455837008 + ], + [ + -164.4309913808565, + 67.61633820257777 + ], + [ + -165.39028683170673, + 68.04277212185025 + ], + [ + -166.76444068099605, + 68.35887685817966 + ], + [ + -166.20470740462667, + 68.88303091091615 + ], + [ + -164.43081051334346, + 68.91553538682774 + ], + [ + -163.1686136546145, + 69.37111481391287 + ], + [ + -162.930566169262, + 69.85806183539927 + ], + [ + -161.90889726463556, + 70.33332998318764 + ], + [ + -160.93479651593367, + 70.44768992784958 + ], + [ + -159.03917578838713, + 70.89164215766891 + ], + [ + -158.11972286683394, + 70.82472117785102 + ], + [ + -156.58082455139808, + 71.35776357694175 + ], + [ + -155.06779029032427, + 71.14777639432367 + ], + [ + -154.3441652089412, + 70.69640859647018 + ], + [ + -153.9000062733926, + 70.88998851183567 + ], + [ + -152.21000606993528, + 70.82999217394485 + ], + [ + -152.27000240782613, + 70.60000621202983 + ], + [ + -150.73999243874448, + 70.43001658800569 + ], + [ + -149.7200030181675, + 70.53001048449045 + ], + [ + -147.61336157935705, + 70.2140349392418 + ], + [ + -145.68998980022533, + 70.12000967068673 + ], + [ + -144.9200109590764, + 69.98999176704046 + ], + [ + -143.58944618042523, + 70.15251414659832 + ], + [ + -142.07251034871348, + 69.85193817817265 + ], + [ + -140.98598752156073, + 69.71199839952635 + ], + [ + -140.98598761037601, + 69.71199839952635 + ] + ] + ], + [ + [ + [ + -171.73165686753944, + 63.782515367275934 + ], + [ + -171.1144335602453, + 63.59219106714495 + ], + [ + -170.4911124339407, + 63.694975490973505 + ], + [ + -169.6825054596536, + 63.43111562769119 + ], + [ + -168.6894394603007, + 63.297506212000556 + ], + [ + -168.77194088445466, + 63.18859813094544 + ], + [ + -169.5294398672051, + 62.97693146427792 + ], + [ + -170.29055620021595, + 63.194437567794424 + ], + [ + -170.67138566799093, + 63.3758218451389 + ], + [ + -171.55306311753873, + 63.317789211675105 + ], + [ + -171.79111060289122, + 63.40584585230046 + ], + [ + -171.73165686753944, + 63.782515367275934 + ] + ] + ] + ] + }, + "properties": { + "continent": "North America", + "gdp_md_est": 18560000, + "iso_a3": "USA", + "name": "United States of America", + "pop_est": 326625791 + }, + "type": "Feature", + "bbox": [ + -171.79111060289122, + 18.91619, + -66.96465999999998, + 71.35776357694175 + ] + } + ] +} \ No newline at end of file diff --git a/internal/geoparquet/featurewriter.go b/internal/geoparquet/featurewriter.go index 5f9325a..04104d8 100644 --- a/internal/geoparquet/featurewriter.go +++ b/internal/geoparquet/featurewriter.go @@ -39,7 +39,7 @@ func NewFeatureWriter(config *WriterConfig) (*FeatureWriter, error) { geoMetadata := config.Metadata if geoMetadata == nil { - geoMetadata = DefaultMetadata() + geoMetadata = DefaultMetadata(config.WriteCoveringMetadata) } if config.ArrowSchema == nil { @@ -100,6 +100,23 @@ func (w *FeatureWriter) append(feature *geo.Feature, field arrow.Field, builder return w.appendGeometry(feature, field, builder) } + if name == DefaultBboxColumn || name == GetBboxColumnNameFromMetadata(w.geoMetadata) { + if feature.Bbox == nil { + if !field.Nullable { + return fmt.Errorf("field %q is required, but the property is missing in the feature", name) + } + builder.AppendNull() + return nil + } + bboxMap := map[string]any{ + "xmin": feature.Bbox.Min.X(), + "ymin": feature.Bbox.Min.Y(), + "xmax": feature.Bbox.Max.X(), + "ymax": feature.Bbox.Max.Y(), + } + return w.appendValue(name, bboxMap, builder) + } + value, ok := feature.Properties[name] if !ok || value == nil { if !field.Nullable { diff --git a/internal/geoparquet/filter.go b/internal/geoparquet/filter.go new file mode 100644 index 0000000..9065c0e --- /dev/null +++ b/internal/geoparquet/filter.go @@ -0,0 +1,298 @@ +package geoparquet + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/compute" + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/metadata" + "github.com/planetlabs/gpq/internal/geo" +) + +// PROJECTION PUSHDOWN - COLUMN FILTERING UTILS + +// A Set type based on map, to hold arrow column indices. +// Implements common Set methods such as Difference() and Contains(). +// To instantiate, use the constructor newIndicesSet() followed by either +// Add() if you want to build the Set sequentially or the convenience function +// FromColNames(). +type indicesSet map[int]struct{} + +func newIndicesSet(size int) *indicesSet { + var s indicesSet = make(map[int]struct{}, size) + return &s +} + +func (s *indicesSet) Add(col int) *indicesSet { + (*s)[col] = struct{}{} + return s +} + +func (s *indicesSet) FromColNames(cols []string, schema *arrow.Schema) *indicesSet { + for _, col := range cols { + if indicesForColumn := schema.FieldIndices(col); indicesForColumn != nil { + for _, colIdx := range indicesForColumn { + s.Add(colIdx) + } + } + } + return s +} + +func (s *indicesSet) Contains(col int) bool { + _, ok := (*s)[col] + return ok +} + +func (s *indicesSet) Difference(other *indicesSet) *indicesSet { + sSize := s.Size() + otherSize := s.Size() + var newSet *indicesSet + if sSize < otherSize { + newSet = newIndicesSet(otherSize - sSize) + } else { + newSet = newIndicesSet(sSize - otherSize) + } + for key := range *s { + if !other.Contains(key) { + newSet.Add(key) + } + } + return newSet +} + +func (s *indicesSet) Size() int { + return len(*s) +} + +func (s *indicesSet) List() []int { + keys := make([]int, 0, len(*s)) + for k := range *s { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// Given a list of columns names to include, return the corresponding columns indices. +func GetColumnIndices(includeColumns []string, arrowSchema *arrow.Schema) ([]int, error) { + // generate indices from col names + indices := newIndicesSet(len(includeColumns)).FromColNames(includeColumns, arrowSchema).List() + + return indices, nil +} + +// Given a list of column names to exclude, return a list of the remaining columns indices. +func GetColumnIndicesByDifference(excludeColumns []string, arrowSchema *arrow.Schema) ([]int, error) { + // generate indices from col names and compute the indices to include + indicesToExclude := newIndicesSet(arrowSchema.NumFields()-len(excludeColumns)).FromColNames(excludeColumns, arrowSchema) + allIndices := newIndicesSet(arrowSchema.NumFields()) + for i := 0; i < arrowSchema.NumFields(); i++ { + allIndices.Add(i) + } + return allIndices.Difference(indicesToExclude).List(), nil +} + +// PREDICATE PUSHDOWN - ROW FILTERING UTILS + +type rowGroupIntersectionResult struct { + Index int + Intersects bool + Error error +} + +// Get row group indices that intersect with the input bbox. Uses the bbox column row group +// stats to calculate intersection. +func GetRowGroupsByBbox(fileReader *file.Reader, bboxCol *BboxColumn, inputBbox *geo.Bbox) ([]int, error) { + numRowGroups := fileReader.NumRowGroups() + intersectingRowGroups := make([]int, 0, numRowGroups) + + // process row groups concurrently + queue := make(chan *rowGroupIntersectionResult) + for i := 0; i < numRowGroups; i += 1 { + go func(i int) { + result := &rowGroupIntersectionResult{Index: i} + result.Intersects, result.Error = RowGroupIntersects(fileReader.MetaData(), bboxCol, i, inputBbox) + queue <- result + }(i) + } + + // read goroutine results + for i := 0; i < numRowGroups; i += 1 { + res := <-queue + if res.Error != nil { + return intersectingRowGroups, res.Error + } + if res.Intersects { + intersectingRowGroups = append(intersectingRowGroups, res.Index) + } + } + slices.Sort(intersectingRowGroups) + return intersectingRowGroups, nil +} + +// Return min/max statistics for a given column and RowGroup. +// For nested structures, use `.`. +func GetColumnMinMax(fileMetadata *metadata.FileMetaData, rowGroup int, columnPath string) (min float64, max float64, err error) { + rowGroupMetadata := fileMetadata.RowGroup(rowGroup) + if rowGroupMetadata == nil { + return 0, 0, fmt.Errorf("metadata for RowGroup %v is nil", rowGroup) + } + + rowGroupSchema := rowGroupMetadata.Schema + if rowGroupSchema == nil { + return 0, 0, fmt.Errorf("schema for RowGroup %v is nil", rowGroup) + } + + columnIdx := rowGroupSchema.ColumnIndexByName(columnPath) + if columnIdx == -1 { + return 0, 0, fmt.Errorf("column %v not found", columnPath) + } + + fieldMetadata, err := rowGroupMetadata.ColumnChunk(columnIdx) + if err != nil { + return 0, 0, fmt.Errorf("couldn't get ColumnChunkMetadata for RowGroup %v/Column %v: %w", rowGroup, columnPath, err) + } + fieldStats, err := fieldMetadata.Statistics() + if err != nil { + return 0, 0, fmt.Errorf("couldn't get ColumnChunkMetadata stats: %w", err) + } + if !fieldStats.HasMinMax() { + return 0, 0, fmt.Errorf("no min/max statistics available for ") + } + + bitsMin := binary.LittleEndian.Uint64(fieldStats.EncodeMin()) + min = math.Float64frombits(bitsMin) + bitsMax := binary.LittleEndian.Uint64(fieldStats.EncodeMax()) + max = math.Float64frombits(bitsMax) + + return min, max, nil +} + +// Check whether the bbox features in a row group intersect with the input bbox, based on the row group min/max stats. +func RowGroupIntersects(fileMetadata *metadata.FileMetaData, bboxCol *BboxColumn, rowGroup int, inputBbox *geo.Bbox) (bool, error) { + if bboxCol.Name == "" { + return false, errors.New("name field of bbox column struct is empty") + } + xminPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Xmin) + xmin, _, err := GetColumnMinMax(fileMetadata, rowGroup, xminPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", xminPath, err) + } + + yminPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Ymin) + ymin, _, err := GetColumnMinMax(fileMetadata, rowGroup, yminPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", yminPath, err) + } + + xmaxPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Xmax) + _, xmax, err := GetColumnMinMax(fileMetadata, rowGroup, xmaxPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", xmaxPath, err) + } + + ymaxPath := fmt.Sprintf("%v.%v", bboxCol.Name, bboxCol.Ymax) + _, ymax, err := GetColumnMinMax(fileMetadata, rowGroup, ymaxPath) + if err != nil { + return false, fmt.Errorf("could not get min/max statistics for %v: %w", ymaxPath, err) + } + + rowGroupBbox := &geo.Bbox{Xmin: xmin, Ymin: ymin, Xmax: xmax, Ymax: ymax} + return rowGroupBbox.Intersects(inputBbox), nil +} + +func filterRecord(ctx context.Context, record *arrow.Record, predicate func(int64) (bool, error)) (*arrow.Record, error) { + // we build a boolean mask and pass it to compute.FilterRecordBatch later + maskBuilder := array.NewBooleanBuilder(memory.DefaultAllocator) + defer maskBuilder.Release() + + // loop over individual bbox values per record + for idx := int64(0); idx < (*record).NumRows(); idx++ { + p, err := predicate(idx) + if err != nil { + return nil, err + } + maskBuilder.Append(p) + } + + r, filterErr := compute.FilterRecordBatch(ctx, *record, maskBuilder.NewBooleanArray(), &compute.FilterOptions{NullSelection: 0}) // TODO check what this is doing + if filterErr != nil { + return nil, fmt.Errorf("trouble filtering record batch: %w", filterErr) + } + return &r, nil +} + +// Filter rows in an arrow.Record by intersection of the feature bounding boxes with an input bbox. +// If there is a bbox column, it will be used to compute intersection. If not, the bbox will be computed +// on the fly. +func FilterRecordBatchByBbox(ctx context.Context, record *arrow.Record, inputBbox *geo.Bbox, bboxCol *BboxColumn) (*arrow.Record, error) { + var filteredRecord *arrow.Record + var filterErr error + + if inputBbox != nil && bboxCol.Index != -1 { // bbox argument has been provided and there is a bbox column we can use for filtering + col := (*record).Column(bboxCol.Index).(*array.Struct) + defer col.Release() + + filteredRecord, filterErr = filterRecord(ctx, record, func(idx int64) (bool, error) { + var bbox map[string]json.RawMessage + if err := json.Unmarshal([]byte(col.ValueStr(int(idx))), &bbox); err != nil { + return false, fmt.Errorf("trouble unmarshalling bbox struct: %w", err) + } + + bboxValue := &geo.Bbox{} // create empty struct to hold bbox values of this row + + if err := json.Unmarshal(bbox[bboxCol.Xmin], &bboxValue.Xmin); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Xmin, err) + } + if err := json.Unmarshal(bbox[bboxCol.Ymin], &bboxValue.Ymin); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Ymin, err) + } + if err := json.Unmarshal(bbox[bboxCol.Xmax], &bboxValue.Xmax); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Xmax, err) + } + if err := json.Unmarshal(bbox[bboxCol.Ymax], &bboxValue.Ymax); err != nil { + return false, fmt.Errorf("trouble parsing bbox.%v field: %w", bboxCol.Ymax, err) + } + + // check whether the bbox passed to this function + // intersects with the bbox of the record + return inputBbox.Intersects(bboxValue), nil + }) + } else if inputBbox != nil && bboxCol.Index == -1 { + // bbox filter passed to function but there is no bbox col. + // this means we have to compute the bboxes of the rows ourselves + primaryColIdx := bboxCol.BaseColumn + col := (*record).Column(primaryColIdx) + defer col.Release() + + filteredRecord, filterErr = filterRecord(ctx, record, func(idx int64) (bool, error) { + value := col.GetOneForMarshal(int(idx)) + g, decodeErr := geo.DecodeGeometry(value, bboxCol.BaseColumnEncoding) + if decodeErr != nil { + return false, fmt.Errorf("trouble decoding geometry: %w", decodeErr) + } + bounds := g.Coordinates.Bound() + bboxValue := &geo.Bbox{ + Xmin: bounds.Min.X(), + Ymin: bounds.Min.Y(), + Xmax: bounds.Max.X(), + Ymax: bounds.Max.Y(), + } + // now that we've computed the bbox, same logic as above + return inputBbox.Intersects(bboxValue), nil + }) + } else { + filteredRecord = record + } + return filteredRecord, filterErr +} diff --git a/internal/geoparquet/filter_test.go b/internal/geoparquet/filter_test.go new file mode 100644 index 0000000..c070154 --- /dev/null +++ b/internal/geoparquet/filter_test.go @@ -0,0 +1,181 @@ +package geoparquet_test + +import ( + "os" + "testing" + + "github.com/apache/arrow/go/v16/arrow/memory" + "github.com/apache/arrow/go/v16/parquet/file" + "github.com/apache/arrow/go/v16/parquet/pqarrow" + "github.com/planetlabs/gpq/internal/geo" + "github.com/planetlabs/gpq/internal/geoparquet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRowGroupIntersects(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} // somewhere in tanzania + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + intersectsEasternHemisphere, err := geoparquet.RowGroupIntersects(fileReader.MetaData(), bboxCol, 0, bbox) + assert.NoError(t, err) + assert.Equal(t, intersectsEasternHemisphere, true) + + intersectsWesternHemisphere, err := geoparquet.RowGroupIntersects(fileReader.MetaData(), bboxCol, 1, bbox) + assert.NoError(t, err) + assert.Equal(t, intersectsWesternHemisphere, false) +} + +func TestGetRowGroupsByBbox(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} // somewhere in tanzania + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.NoError(t, err) + + // only the eastern hemisphere row group matches + require.Len(t, rowGroups, 1) + assert.Equal(t, 0, rowGroups[0]) +} + +func TestGetRowGroupsByBbox2(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: -92.0, Ymin: 32.0, Xmax: -88.0, Ymax: 35.0} // somewhere in louisiana + geoMetadata, err := geoparquet.GetMetadataFromFileReader(fileReader) + require.NoError(t, err) + + bboxCol := geoparquet.GetBboxColumn(fileReader.MetaData().Schema, geoMetadata) + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.NoError(t, err) + + // only the western hemisphere row group matches + require.Len(t, rowGroups, 1) + assert.Equal(t, 1, rowGroups[0]) +} + +func TestGetRowGroupsByBboxErrorNoBboxCol(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + bbox := &geo.Bbox{Xmin: -92.0, Ymin: 32.0, Xmax: -88.0, Ymax: 35.0} // somewhere in louisiana + + bboxCol := &geoparquet.BboxColumn{} // empty bbox col, will raise error + + // the file has two row groups - the first one contains all data for the eastern hemisphere, + // the second for the western hemisphere + rowGroups, err := geoparquet.GetRowGroupsByBbox(fileReader, bboxCol, bbox) + require.ErrorContains(t, err, "bbox column") + assert.Empty(t, rowGroups) +} + +func TestGetColumnMinMax(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0-partitioned.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + require.Equal(t, 2, fileReader.NumRowGroups()) + + xminMin, xminMax, err := geoparquet.GetColumnMinMax(fileReader.MetaData(), 0, "bbox.xmin") + assert.NoError(t, err) + assert.Equal(t, 29.339997592900346, xminMin) + assert.Equal(t, 29.339997592900346, xminMax) + + xmaxMin, xmaxMax, err := geoparquet.GetColumnMinMax(fileReader.MetaData(), 0, "bbox.xmax") + assert.NoError(t, err) + assert.Equal(t, 40.31659000000002, xmaxMin) + assert.Equal(t, 40.31659000000002, xmaxMax) + + xminMin, xminMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmin") + assert.NoError(t, err) + assert.Equal(t, -171.79111060289122, xminMin) + assert.Equal(t, -17.06342322434257, xminMax) + + xmaxMin, xmaxMax, err = geoparquet.GetColumnMinMax(fileReader.MetaData(), 1, "bbox.xmax") + assert.NoError(t, err) + assert.Equal(t, -66.96465999999998, xmaxMin) + assert.Equal(t, -8.665124477564191, xmaxMax) +} + +func TestGetColumnIndices(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + arrowReader, err := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + require.NoError(t, err) + schema, err := arrowReader.Schema() + require.NoError(t, err) + + indices, err := geoparquet.GetColumnIndices([]string{"pop_est", "name", "iso_a3"}, schema) + assert.NoError(t, err) + assert.Equal(t, []int{0, 2, 3}, indices) +} + +func TestGetColumnIndicesByDifference(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + fileReader, err := file.NewParquetReader(input) + require.NoError(t, err) + + arrowReader, err := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: 1024}, memory.DefaultAllocator) + require.NoError(t, err) + schema, err := arrowReader.Schema() + require.NoError(t, err) + + indices, err := geoparquet.GetColumnIndicesByDifference([]string{"pop_est", "name", "iso_a3"}, schema) + assert.NoError(t, err) + assert.Equal(t, []int{1, 4, 5, 6}, indices) +} diff --git a/internal/geoparquet/geoparquet.go b/internal/geoparquet/geoparquet.go index faaa6a3..4c884ed 100644 --- a/internal/geoparquet/geoparquet.go +++ b/internal/geoparquet/geoparquet.go @@ -189,3 +189,76 @@ func FromParquet(input parquet.ReaderAtSeeker, output io.Writer, convertOptions return pqutil.TransformByColumn(config) } + +type BboxColumnFieldNames struct { + Xmin string + Ymin string + Xmax string + Ymax string +} + +func GetBboxColumnFieldNames(metadata *Metadata) *BboxColumnFieldNames { + // infer bbox struct field names + fieldNames := &BboxColumnFieldNames{} + + if metadata.Columns[metadata.PrimaryColumn].Covering != nil { + fieldNames.Xmin = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmin[1] + fieldNames.Ymin = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymin[1] + fieldNames.Xmax = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Xmax[1] + fieldNames.Ymax = metadata.Columns[metadata.PrimaryColumn].Covering.Bbox.Ymax[1] + } else { + // fallback to standard names + fieldNames.Xmin = "xmin" + fieldNames.Ymin = "ymin" + fieldNames.Xmax = "xmax" + fieldNames.Ymax = "ymax" + } + + return fieldNames +} + +type BboxColumn struct { + Index int + Name string + BaseColumn int // the primary geometry column the bbox column references + BaseColumnEncoding string + BboxColumnFieldNames +} + +// Get Bbox column name from covering metadata only. For most cases, use the +// more robust function GetBboxColumn that infers the name even if metadata +// is not present. +func GetBboxColumnNameFromMetadata(geoMetadata *Metadata) string { + if geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering != nil && len(geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { + return geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin[0] + } + return "" +} + +// Returns a *BboxColumn struct that contains index, name and other data +// that describe the bounding box column contained in the schema. +// If there is no match for the standard name "bbox" in the schema, +// the covering metadata is consulted. +// An index field value of -1 (alongside an empty name field) means no bbox column found. +func GetBboxColumn(schema *schema.Schema, geoMetadata *Metadata) *BboxColumn { + bboxCol := &BboxColumn{} + // try standard name first + bboxCol.Name = DefaultBboxColumn + + // NB: we can't do schema.ColumnIndexByName() in this case as it won't give the expected result for nested types like structs + bboxCol.Index = schema.Root().FieldIndexByName(DefaultBboxColumn) + + // if no match, check covering metadata + if bboxCol.Index == -1 { + if geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering != nil && len(geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin) == 2 { + bboxCol.Name = geoMetadata.Columns[geoMetadata.PrimaryColumn].Covering.Bbox.Xmin[0] + bboxCol.Index = schema.Root().FieldIndexByName(bboxCol.Name) + } else { + bboxCol.Name = "" + } + } + + bboxCol.BaseColumn = schema.Root().FieldIndexByName(geoMetadata.PrimaryColumn) + bboxCol.BboxColumnFieldNames = *GetBboxColumnFieldNames(geoMetadata) + return bboxCol +} diff --git a/internal/geoparquet/geoparquet_test.go b/internal/geoparquet/geoparquet_test.go index f6e5a5a..a643ed7 100644 --- a/internal/geoparquet/geoparquet_test.go +++ b/internal/geoparquet/geoparquet_test.go @@ -108,7 +108,7 @@ func TestRecordReaderV040(t *testing.T) { input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: input, }) require.NoError(t, err) @@ -131,12 +131,13 @@ func TestRowReaderV100Beta1(t *testing.T) { input, openErr := os.Open(fixturePath) require.NoError(t, openErr) - reader, err := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ Reader: input, }) require.NoError(t, err) numRows := 0 + var numCols int for { record, err := reader.Read() if err == io.EOF { @@ -144,9 +145,72 @@ func TestRowReaderV100Beta1(t *testing.T) { } require.NoError(t, err) numRows += int(record.NumRows()) + numCols = int(record.NumCols()) } assert.Equal(t, 5, numRows) + assert.Equal(t, 6, numCols) +} + +func TestRecordReaderV100Columns(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.0.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{0, 1, 4, 5}, + }) + require.NoError(t, err) + + record, err := reader.Read() + require.NoError(t, err) + assert.Equal(t, record.NumCols(), int64(4)) + + fields := record.Schema().Fields() + colNames := make([]string, len(fields)) + for idx, field := range fields { + colNames[idx] = field.Name + } + + assert.ElementsMatch(t, colNames, []string{"geometry", "pop_est", "iso_a3", "name"}) +} + +func TestRecordReaderV110Columns(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.0.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{0, 2, 3}, + }) + require.NoError(t, err) + + record, err := reader.Read() + require.NoError(t, err) + assert.Equal(t, record.NumCols(), int64(3)) + + fields := record.Schema().Fields() + colNames := make([]string, len(fields)) + for idx, field := range fields { + colNames[idx] = field.Name + } + + assert.ElementsMatch(t, colNames, []string{"geometry", "continent", "gdp_md_est"}) +} + +func TestRecordReaderV110NoGeomColError(t *testing.T) { + fixturePath := "../testdata/cases/example-v1.1.0.parquet" + input, openErr := os.Open(fixturePath) + require.NoError(t, openErr) + + reader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ + Reader: input, + Columns: []int{2, 3}, + }) + require.ErrorContains(t, err, "geometry column") + require.Nil(t, reader) } func toWKB(t *testing.T, geometry orb.Geometry) []byte { @@ -213,7 +277,7 @@ func TestFromParquetWithoutDefaultGeometryColumn(t *testing.T) { } func TestMetadataClone(t *testing.T) { - metadata := geoparquet.DefaultMetadata() + metadata := geoparquet.DefaultMetadata(false) clone := metadata.Clone() assert.Equal(t, metadata.PrimaryColumn, clone.PrimaryColumn) @@ -394,3 +458,181 @@ func TestRecordReading(t *testing.T) { assert.Equal(t, reader.NumRows(), int64(numRows)) } + +func TestGetBboxColumnV100(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // no bbox col in the file, we expect -1 + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, -1, bboxCol.Index) + assert.Equal(t, "", bboxCol.Name) + assert.Equal(t, 0, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) +} + +func TestGetBboxColumnV110(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // there is a bbox col in the file, we expect index 6 + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, 6, bboxCol.Index) + assert.Equal(t, "bbox", bboxCol.Name) + assert.Equal(t, 5, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) +} + +func TestGetBboxColumnIdxV110NonStandardBboxCol(t *testing.T) { + f, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") + require.NoError(t, fileErr) + reader, readerErr := file.NewParquetReader(f) + require.NoError(t, readerErr) + defer reader.Close() + + metadata, err := geoparquet.GetMetadata(reader.MetaData().KeyValueMetadata()) + require.NoError(t, err) + + // there is a bbox col in the file with the non-standard name "geometry_bbox", + // we expect index 6 + bboxCol := geoparquet.GetBboxColumn(reader.MetaData().Schema, metadata) + assert.Equal(t, 6, bboxCol.Index) + assert.Equal(t, "geometry_bbox", bboxCol.Name) + assert.Equal(t, 5, bboxCol.BaseColumn) + assert.Equal(t, "", bboxCol.BaseColumnEncoding) + assert.Equal(t, "xmin", bboxCol.BboxColumnFieldNames.Xmin) + assert.Equal(t, "ymin", bboxCol.BboxColumnFieldNames.Ymin) + assert.Equal(t, "xmax", bboxCol.BboxColumnFieldNames.Xmax) + assert.Equal(t, "ymax", bboxCol.BboxColumnFieldNames.Ymax) + assert.Equal(t, 6, reader.MetaData().Schema.Root().FieldIndexByName("geometry_bbox")) +} + +func TestFilterRecordBatchByBboxV100(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.0.0.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(6), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: -1, + BaseColumn: 0, + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(6), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} + +func TestFilterRecordBatchByBboxV110(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(7), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: 6, + BaseColumn: 5, + BaseColumnEncoding: "wkb", + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(7), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} + +func TestFilterRecordBatchByBboxV110NonStandardBboxCol(t *testing.T) { + fileReader, fileErr := os.Open("../testdata/cases/example-v1.1.0-covering.parquet") + require.NoError(t, fileErr) + defer fileReader.Close() + + recordReader, err := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{Reader: fileReader}) + require.NoError(t, err) + defer recordReader.Close() + + record, readErr := recordReader.Read() + require.NoError(t, readErr) + assert.Equal(t, int64(7), record.NumCols()) + assert.Equal(t, int64(5), record.NumRows()) + + inputBbox := &geo.Bbox{Xmin: 34.0, Ymin: -7.0, Xmax: 36.0, Ymax: -6.0} + + filteredRecord, err := geoparquet.FilterRecordBatchByBbox(context.Background(), &record, inputBbox, &geoparquet.BboxColumn{ + Index: 6, + BaseColumn: 5, + BaseColumnEncoding: "wkb", + BboxColumnFieldNames: geoparquet.BboxColumnFieldNames{ + Xmin: "xmin", + Ymin: "ymin", + Xmax: "xmax", + Ymax: "ymax", + }, + }) + require.NoError(t, err) + + // we expect only one row, namely Tanzania + assert.Equal(t, int64(7), (*filteredRecord).NumCols()) + assert.Equal(t, int64(1), (*filteredRecord).NumRows()) + + country := (*filteredRecord).Column(recordReader.Schema().ColumnIndexByName("name")).ValueStr(0) + assert.Equal(t, "Tanzania", country) +} diff --git a/internal/geoparquet/metadata.go b/internal/geoparquet/metadata.go index 506aa09..7b9e38b 100644 --- a/internal/geoparquet/metadata.go +++ b/internal/geoparquet/metadata.go @@ -4,17 +4,19 @@ import ( "encoding/json" "fmt" + "github.com/apache/arrow/go/v16/parquet/file" "github.com/apache/arrow/go/v16/parquet/metadata" "github.com/planetlabs/gpq/internal/geo" ) const ( - Version = "1.0.0" + Version = "1.1.0" MetadataKey = "geo" EdgesPlanar = "planar" EdgesSpherical = "spherical" OrientationCounterClockwise = "counterclockwise" DefaultGeometryColumn = "geometry" + DefaultBboxColumn = "bbox" DefaultGeometryEncoding = geo.EncodingWKB ) @@ -79,6 +81,17 @@ func (p *Proj) String() string { return id } +type coveringBbox struct { + Xmin []string + Ymin []string + Xmax []string + Ymax []string +} + +type Covering struct { + Bbox coveringBbox +} + type GeometryColumn struct { Encoding string `json:"encoding"` GeometryType any `json:"geometry_type,omitempty"` @@ -88,6 +101,7 @@ type GeometryColumn struct { Orientation string `json:"orientation,omitempty"` Bounds []float64 `json:"bbox,omitempty"` Epoch float64 `json:"epoch,omitempty"` + Covering *Covering `json:"covering,omitempty"` } func (g *GeometryColumn) clone() *GeometryColumn { @@ -139,14 +153,23 @@ func getDefaultGeometryColumn() *GeometryColumn { } } -func DefaultMetadata() *Metadata { - return &Metadata{ +func DefaultMetadata(writeCovering bool) *Metadata { + metadata := &Metadata{ Version: Version, PrimaryColumn: DefaultGeometryColumn, Columns: map[string]*GeometryColumn{ DefaultGeometryColumn: getDefaultGeometryColumn(), }, } + if writeCovering { + metadata.Columns[metadata.PrimaryColumn].Covering = &Covering{Bbox: coveringBbox{ + Xmin: []string{"bbox", "xmin"}, + Ymin: []string{"bbox", "ymin"}, + Xmax: []string{"bbox", "xmax"}, + Ymax: []string{"bbox", "ymax"}, + }} + } + return metadata } var ErrNoMetadata = fmt.Errorf("missing %s metadata key", MetadataKey) @@ -180,3 +203,7 @@ func GetMetadataValue(keyValueMetadata metadata.KeyValueMetadata) (string, error } return *value, nil } + +func GetMetadataFromFileReader(fileReader *file.Reader) (*Metadata, error) { + return GetMetadata(fileReader.MetaData().GetKeyValueMetadata()) +} diff --git a/internal/geoparquet/recordreader.go b/internal/geoparquet/recordreader.go index 78979c8..b2e6f5e 100644 --- a/internal/geoparquet/recordreader.go +++ b/internal/geoparquet/recordreader.go @@ -3,6 +3,8 @@ package geoparquet import ( "context" "errors" + "fmt" + "slices" "github.com/apache/arrow/go/v16/arrow" "github.com/apache/arrow/go/v16/arrow/memory" @@ -21,6 +23,8 @@ type ReaderConfig struct { Reader parquet.ReaderAtSeeker File *file.Reader Context context.Context + Columns []int + RowGroups []int } type RecordReader struct { @@ -29,46 +33,106 @@ type RecordReader struct { recordReader pqarrow.RecordReader } -func NewRecordReader(config *ReaderConfig) (*RecordReader, error) { +func NewParquetFileReader(config *ReaderConfig) (*file.Reader, error) { + fileReader := config.File + if fileReader == nil { + if config.Reader == nil { + return nil, errors.New("config must include a File or Reader value") + } + fr, frErr := file.NewParquetReader(config.Reader) + if frErr != nil { + return nil, frErr + } + fileReader = fr + } + return fileReader, nil +} + +func NewArrowFileReader(config *ReaderConfig, parquetReader *file.Reader) (*pqarrow.FileReader, error) { batchSize := config.BatchSize if batchSize == 0 { batchSize = defaultReadBatchSize } + return pqarrow.NewFileReader(parquetReader, pqarrow.ArrowReadProperties{BatchSize: int64(batchSize)}, memory.DefaultAllocator) +} + +func NewRecordReaderFromConfig(config *ReaderConfig) (*RecordReader, error) { + parquetFileReader, err := NewParquetFileReader(config) + if err != nil { + return nil, fmt.Errorf("could not get ParquetFileReader: %w", err) + } + + arrowFileReader, err := NewArrowFileReader(config, parquetFileReader) + if err != nil { + return nil, fmt.Errorf("could not get ArrowFileReader: %w", err) + } + + geoMetadata, err := GetMetadataFromFileReader(parquetFileReader) + if err != nil { + return nil, fmt.Errorf("could not get geo metadata from file reader: %w", err) + } + ctx := config.Context if ctx == nil { ctx = context.Background() } - fileReader := config.File - if fileReader == nil { - if config.Reader == nil { - return nil, errors.New("config must include a File or Reader value") + if config.Columns != nil { + primaryGeomColIdx := parquetFileReader.MetaData().Schema.ColumnIndexByName(geoMetadata.PrimaryColumn) + + if !slices.Contains(config.Columns, primaryGeomColIdx) { + return nil, fmt.Errorf("columns must include primary geometry column '%v' (index %v)", geoMetadata.PrimaryColumn, primaryGeomColIdx) } - fr, frErr := file.NewParquetReader(config.Reader) - if frErr != nil { - return nil, frErr + } + + if config.Columns != nil && len(config.Columns) == 0 { + config.Columns = nil + } + + if config.RowGroups != nil && len(config.RowGroups) == 0 { + config.RowGroups = nil + } + + recordReader, recordErr := arrowFileReader.GetRecordReader(ctx, config.Columns, config.RowGroups) + + if recordErr != nil { + return nil, recordErr + } + + reader := &RecordReader{ + fileReader: arrowFileReader.ParquetReader(), + metadata: geoMetadata, + recordReader: recordReader, + } + return reader, nil +} + +func NewRecordReader(ctx context.Context, arrowFileReader *pqarrow.FileReader, geoMetadata *Metadata, columns []int, rowGroups []int) (*RecordReader, error) { + if columns != nil || len(columns) != 0 { + primaryGeomColIdx := arrowFileReader.ParquetReader().MetaData().Schema.ColumnIndexByName(geoMetadata.PrimaryColumn) + + if !slices.Contains(columns, primaryGeomColIdx) { + return nil, fmt.Errorf("columns (%v) must include primary geometry column '%v' (index %v)", columns, geoMetadata.PrimaryColumn, primaryGeomColIdx) } - fileReader = fr } - geoMetadata, geoMetadataErr := GetMetadata(fileReader.MetaData().GetKeyValueMetadata()) - if geoMetadataErr != nil { - return nil, geoMetadataErr + if columns != nil && len(columns) == 0 { + columns = nil } - arrowReader, arrowErr := pqarrow.NewFileReader(fileReader, pqarrow.ArrowReadProperties{BatchSize: int64(batchSize)}, memory.DefaultAllocator) - if arrowErr != nil { - return nil, arrowErr + if rowGroups != nil && len(rowGroups) == 0 { + rowGroups = nil } - recordReader, recordErr := arrowReader.GetRecordReader(ctx, nil, nil) + recordReader, recordErr := arrowFileReader.GetRecordReader(ctx, columns, rowGroups) + if recordErr != nil { return nil, recordErr } reader := &RecordReader{ - fileReader: fileReader, + fileReader: arrowFileReader.ParquetReader(), metadata: geoMetadata, recordReader: recordReader, } @@ -87,6 +151,14 @@ func (r *RecordReader) Schema() *schema.Schema { return r.fileReader.MetaData().Schema } +func (r *RecordReader) ArrowSchema() *arrow.Schema { + return r.recordReader.Schema() +} + +func (r *RecordReader) NumRows() int64 { + return r.fileReader.NumRows() +} + func (r *RecordReader) Close() error { r.recordReader.Release() return r.fileReader.Close() diff --git a/internal/geoparquet/recordwriter.go b/internal/geoparquet/recordwriter.go index e4f3ab2..d4ceed7 100644 --- a/internal/geoparquet/recordwriter.go +++ b/internal/geoparquet/recordwriter.go @@ -11,9 +11,10 @@ import ( ) type RecordWriter struct { - fileWriter *pqarrow.FileWriter - metadata *Metadata - wroteGeoMetadata bool + fileWriter *pqarrow.FileWriter + metadata *Metadata + wroteGeoMetadata bool + writeCoveringMetadata bool } func NewRecordWriter(config *WriterConfig) (*RecordWriter, error) { @@ -41,8 +42,9 @@ func NewRecordWriter(config *WriterConfig) (*RecordWriter, error) { } writer := &RecordWriter{ - fileWriter: fileWriter, - metadata: config.Metadata, + fileWriter: fileWriter, + metadata: config.Metadata, + writeCoveringMetadata: config.WriteCoveringMetadata, } return writer, nil @@ -66,7 +68,7 @@ func (w *RecordWriter) Close() error { if !w.wroteGeoMetadata { metadata := w.metadata if metadata == nil { - metadata = DefaultMetadata() + metadata = DefaultMetadata(w.writeCoveringMetadata) } data, err := json.Marshal(metadata) if err != nil { diff --git a/internal/geoparquet/writer.go b/internal/geoparquet/writer.go index 5554118..11e3bc3 100644 --- a/internal/geoparquet/writer.go +++ b/internal/geoparquet/writer.go @@ -9,9 +9,10 @@ import ( ) type WriterConfig struct { - Writer io.Writer - Metadata *Metadata - ParquetWriterProps *parquet.WriterProperties - ArrowWriterProps *pqarrow.ArrowWriterProperties - ArrowSchema *arrow.Schema + Writer io.Writer + Metadata *Metadata + ParquetWriterProps *parquet.WriterProperties + ArrowWriterProps *pqarrow.ArrowWriterProperties + ArrowSchema *arrow.Schema + WriteCoveringMetadata bool } diff --git a/internal/pqutil/arrow.go b/internal/pqutil/arrow.go index 0f20c98..b68c5b4 100644 --- a/internal/pqutil/arrow.go +++ b/internal/pqutil/arrow.go @@ -39,6 +39,17 @@ func (b *ArrowSchemaBuilder) AddGeometry(name string, encoding string) error { return nil } +func (b *ArrowSchemaBuilder) AddBbox(name string) { + bboxFields := []arrow.Field{ + {Name: "xmin", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "ymin", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "xmax", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + {Name: "ymax", Type: arrow.PrimitiveTypes.Float64, Nullable: false}, + } + dataType := arrow.StructOf(bboxFields...) + b.fields[name] = &arrow.Field{Name: name, Type: dataType, Nullable: true} +} + func (b *ArrowSchemaBuilder) Add(record map[string]any) error { for name, value := range record { if b.fields[name] != nil { diff --git a/internal/testdata/cases/example-v1.1.0-covering.parquet b/internal/testdata/cases/example-v1.1.0-covering.parquet new file mode 100644 index 0000000..f831180 Binary files /dev/null and b/internal/testdata/cases/example-v1.1.0-covering.parquet differ diff --git a/internal/testdata/cases/example-v1.1.0-partitioned.parquet b/internal/testdata/cases/example-v1.1.0-partitioned.parquet new file mode 100644 index 0000000..42084f9 Binary files /dev/null and b/internal/testdata/cases/example-v1.1.0-partitioned.parquet differ diff --git a/internal/testdata/cases/example-v1.1.0.parquet b/internal/testdata/cases/example-v1.1.0.parquet new file mode 100644 index 0000000..c025304 Binary files /dev/null and b/internal/testdata/cases/example-v1.1.0.parquet differ diff --git a/internal/validator/validator.go b/internal/validator/validator.go index a68df1f..b9705ba 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -169,7 +169,7 @@ func (v *Validator) Report(ctx context.Context, file *file.Reader) (*Report, err } // run all the data scanning rules - recordReader, rrErr := geoparquet.NewRecordReader(&geoparquet.ReaderConfig{ + recordReader, rrErr := geoparquet.NewRecordReaderFromConfig(&geoparquet.ReaderConfig{ File: file, Context: ctx, }) diff --git a/readme.md b/readme.md index 542d7c0..049b2ad 100644 --- a/readme.md +++ b/readme.md @@ -82,6 +82,18 @@ The `describe` command prints schema information and metadata about a GeoParquet gpq describe example.parquet ``` +### extract + +The `extract` command can be use to extract columns and/or rows from a local or remote GeoParquet file. + +``` +gpq extract input.parquet output.parquet --bbox=xmin,ymin,xmax,ymax --drop-cols=col1,col2 +``` + +Instead of negatively selecting columns by specifying which ones to drop (`--drop-cols`), you can alternatively use the `--keep-only-cols` argument to explicitely select those columns that you wish to keep in the data set. + +The `--bbox` argument allows you to extract features whose bounding box intersects with the provided bbox. Note that this doesn't support exact geometry filtering and will only operate on bounding boxes of full feature geometries. It is thus recommended to use the `--bbox` argument for preliminary filtering only. The algorithm will attempt to use an existing bounding box column in the file. If bounding box information is not available, the bounding boxes will be computed on the fly. If the GeoParquet file is spatially partitioned using row groups, the algorithm will use row group statistics to speed up the filtering process. + ## Limitations * Non-geographic CRS information is not preserved when converting GeoParquet to GeoJSON.