feat: ignoreoutsidegrid (#36)

add flag to ignore-outside-grid polygons

also
* default keep-points-and-lines to false
* add gdal in the docker image

---------

Co-authored-by: kad-dirc <[email protected]>
Co-authored-by: bergr <[email protected]>

Dockerfile

@@ -1,3 +1,6 @@
+ARG GDAL_VERSION=3.6.3
+
+FROM ghcr.io/osgeo/gdal:ubuntu-small-${GDAL_VERSION} AS base
 FROM golang:1.21-bullseye AS build-env
 
 ENV GO111MODULE=on
@@ -25,18 +28,16 @@ RUN go test ./... -covermode=atomic
 
 RUN go build -v -buildvcs=true -ldflags='-s -w -linkmode auto' -a -installsuffix cgo -o /texel .
 
-# FROM scratch
-FROM golang:1.21-bullseye
+FROM base AS build-image
+
 RUN apt-get update && apt-get install -y \
   libsqlite3-mod-spatialite \
   && rm -rf /var/lib/apt/lists/*
 
 # important for time conversion
 ENV TZ Europe/Amsterdam
-
 WORKDIR /
 
-# Import from builder.
-COPY --from=build-env /texel /
+COPY --from=build-env /texel /usr/local/bin/texel
 
-ENTRYPOINT ["/texel"]
+ENTRYPOINT ["texel"]

main.go

@@ -32,6 +32,7 @@ const TILEMATRIXSET string = `tilematrixset`
 const TILEMATRICES string = `tilematrices`
 const PAGESIZE string = `pagesize`
 const KEEPPOINTSANDLINES string = `keeppointsandlines`
+const IGNOREOUTSIDEGRID string = `ignoreoutsidegrid`
 
 //nolint:funlen
 func main() {
@@ -85,13 +86,22 @@ func main() {
 			EnvVars:  []string{strcase.ToScreamingSnake(PAGESIZE)},
 		},
 		&cli.BoolFlag{
-			Name:     KEEPPOINTSANDLINES,
-			Aliases:  []string{"pl"},
-			Usage:    "Parts of polygons are reduced to points and lines after texel, keep these details or not.",
-			Value:    true,
+			Name:    KEEPPOINTSANDLINES,
+			Aliases: []string{"pl"},
+			Usage:   "Parts of polygons are reduced to points and lines after texel, keep these details or not.",
+			// TODO: This results in broken tiles, we should change this to a collapse option (optionally collapse rings from polygons to lines and points; grow lines as long as possible and add an option for line-length-to-keep)
+			Value:    false,
 			Required: false,
 			EnvVars:  []string{strcase.ToScreamingSnake(KEEPPOINTSANDLINES)},
 		},
+		&cli.BoolFlag{
+			Name:     IGNOREOUTSIDEGRID,
+			Aliases:  []string{"iog"},
+			Usage:    "Ignore polygons that fall (partly) outside the grid, instead of panicking",
+			Value:    false,
+			Required: false,
+			EnvVars:  []string{strcase.ToScreamingSnake(IGNOREOUTSIDEGRID)},
+		},
 	}
 
 	app.Action = func(c *cli.Context) error {
@@ -122,7 +132,10 @@ func main() {
 		gpkgTargets := make(map[int]*gpkg.TargetGeopackage, len(tileMatrixIDs))
 		overwrite := c.Bool(OVERWRITE)
 		pagesize := c.Int(PAGESIZE) // TODO divide by tile matrices count
-		keepPointsAndLines := c.Bool(KEEPPOINTSANDLINES)
+		snapConfig := snap.Config{
+			KeepPointsAndLines: c.Bool(KEEPPOINTSANDLINES),
+			IgnoreOutsideGrid:  c.Bool(IGNOREOUTSIDEGRID),
+		}
 		for _, tmID := range tileMatrixIDs {
 			gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize)
 			defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func
@@ -150,7 +163,7 @@ func main() {
 				source.Table = table
 				target.Table = table
 			}
-			processBySnapping(source, targets, tileMatrixSet, keepPointsAndLines)
+			processBySnapping(source, targets, tileMatrixSet, snapConfig)
 			log.Printf("  finished %s", table.Name)
 		}
 
@@ -171,7 +184,7 @@ func validateTileMatrixSet(tms tms20.TileMatrixSet, tileMatrixIDs []tms20.TMID)
 		return err
 	}
 	if deviationInPixels >= 1 {
-		log.Printf("warning, (largest) deviation is larger than 1 tile pixel (%f units) on the deepest matrix (%d)\n", deviationInUnits, deepestTMID)
+		log.Printf("[WARNING] (largest) deviation is larger than 1 tile pixel (%f units) on the deepest matrix (%d)\n", deviationInUnits, deepestTMID)
 		log.Println(stats)
 	}
 	return pointindex.IsQuadTree(tms)
@@ -200,8 +213,8 @@ func injectSuffixIntoPath(p string) string {
 	return path.Join(dir, name+"_%v"+ext)
 }
 
-func processBySnapping(source processing.Source, targets map[tms20.TMID]processing.Target, tileMatrixSet tms20.TileMatrixSet, keepPointsAndLines bool) {
+func processBySnapping(source processing.Source, targets map[tms20.TMID]processing.Target, tileMatrixSet tms20.TileMatrixSet, snapConfig snap.Config) {
 	processing.ProcessFeatures(source, targets, func(p geom.Polygon, tmIDs []tms20.TMID) map[tms20.TMID][]geom.Polygon {
-		return snap.SnapPolygon(p, tileMatrixSet, tmIDs, keepPointsAndLines)
+		return snap.SnapPolygon(p, tileMatrixSet, tmIDs, snapConfig)
 	})
 }

pointindex/pointindex.go

@@ -107,7 +107,7 @@ func FromTileMatrixSet(tileMatrixSet tms20.TileMatrixSet, deepestTMID tms20.TMID
 }
 
 // InsertPolygon inserts all points from a Polygon
-func (ix *PointIndex) InsertPolygon(polygon geom.Polygon) {
+func (ix *PointIndex) InsertPolygon(polygon geom.Polygon) error {
 	// initialize the quadrants map
 	pointsCount := 0
 	for _, ring := range polygon.LinearRings() {
@@ -118,31 +118,47 @@ func (ix *PointIndex) InsertPolygon(polygon geom.Polygon) {
 		if ix.quadrants[level] == nil {
 			ix.quadrants[level] = make(map[morton.Z]Quadrant, pointsCount) // TODO smaller for the shallower levels
 		}
-		// TODO expand for another polygon?
 	}
 
 	for _, ring := range polygon.LinearRings() {
 		for _, vertex := range ring {
-			ix.InsertPoint(vertex)
+			if err := ix.InsertPoint(vertex); err != nil {
+				return err
+			}
 		}
 	}
+	return nil
 }
 
 // InsertPoint inserts a Point by its absolute coord
-func (ix *PointIndex) InsertPoint(point geom.Point) {
+func (ix *PointIndex) InsertPoint(point geom.Point) error {
 	intPoint := intgeom.FromGeomPoint(point)
 	deepestX := int((intPoint.X() - ix.intExtent.MinX()) / ix.deepestRes)
 	deepestY := int((intPoint.Y() - ix.intExtent.MinY()) / ix.deepestRes)
-	ix.InsertCoord(deepestX, deepestY)
+	return ix.InsertCoord(deepestX, deepestY)
+}
+
+type OutsideGridError struct {
+	deepestX    int
+	deepestY    int
+	deepestSize uint
+}
+
+func (e OutsideGridError) Error() string {
+	return fmt.Sprintf("trying to insert a coord (%v, %v) outside the grid/extent (0, %v; 0, %v)", e.deepestX, e.deepestY, e.deepestSize, e.deepestSize)
 }
 
 // InsertCoord inserts a Point by its x/y coord on the deepest level
-func (ix *PointIndex) InsertCoord(deepestX int, deepestY int) {
+func (ix *PointIndex) InsertCoord(deepestX int, deepestY int) error {
 	if deepestX < 0 || deepestY < 0 || deepestX > int(ix.deepestSize)-1 || deepestY > int(ix.deepestSize)-1 {
-		// should never happen
-		panic(fmt.Errorf("trying to insert a coord (%v, %v) outside the grid/extent (0, %v; 0, %v)", deepestX, deepestY, ix.deepestSize, ix.deepestSize))
+		return OutsideGridError{
+			deepestX:    deepestX,
+			deepestY:    deepestY,
+			deepestSize: ix.deepestSize,
+		}
 	}
 	ix.insertCoord(deepestX, deepestY)
+	return nil
 }
 
 // insertCoord adds a point into this pc, assuming the point is inside its extent

pointindex/pointindex_test.go

@@ -268,7 +268,8 @@ func TestPointIndex_InsertPoint(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			ix := tt.ix
-			ix.InsertPoint(tt.point)
+			err := ix.InsertPoint(tt.point)
+			require.NoError(t, err)
 			if tt.want.hitOnce == nil {
 				tt.want.hitOnce = make(map[morton.Z]map[intgeom.Point][]int)
 			}
@@ -326,9 +327,10 @@ func TestPointIndex_InsertPoint_Deepest(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			tms := loadEmbeddedTileMatrixSet(t, tt.tmsID)
 			ix, err := FromTileMatrixSet(tms, tt.tmID)
-			require.Nil(t, err)
+			require.NoError(t, err)
 
-			ix.InsertPoint(tt.point)
+			err = ix.InsertPoint(tt.point)
+			require.NoError(t, err)
 			assert.Equal(t, 1, len(ix.quadrants[ix.deepestLevel]))
 			for z, quadrant := range ix.quadrants[ix.deepestLevel] {
 				assert.EqualValues(t, tt.want, quadrant)
@@ -467,7 +469,8 @@ func TestPointIndex_SnapClosestPoints(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			ix := tt.ix
 			poly := tt.poly
-			ix.InsertPolygon(poly)
+			err := ix.InsertPolygon(poly)
+			require.NoError(t, err)
 			levels := tt.levels
 			if levels == nil {
 				levels = []Level{ix.deepestLevel}

snap/snap.go

@@ -1,6 +1,7 @@
 package snap
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"math"
@@ -29,11 +30,16 @@ const (
 
 type IsOuter = bool
 
+type Config struct {
+	KeepPointsAndLines bool
+	IgnoreOutsideGrid  bool
+}
+
 // SnapPolygon snaps polygons' points to a tile's internal pixel grid
 // and adds points to lines to prevent intersections.
 //
 //nolint:revive
-func SnapPolygon(polygon geom.Polygon, tileMatrixSet tms20.TileMatrixSet, tmIDs []tms20.TMID, keepPointsAndLines bool) map[tms20.TMID][]geom.Polygon {
+func SnapPolygon(polygon geom.Polygon, tileMatrixSet tms20.TileMatrixSet, tmIDs []tms20.TMID, config Config) map[tms20.TMID][]geom.Polygon {
 	deepestID := slices.Max(tmIDs)
 	ix, err := pointindex.FromTileMatrixSet(tileMatrixSet, deepestID)
 	if err != nil {
@@ -45,8 +51,18 @@ func SnapPolygon(polygon geom.Polygon, tileMatrixSet tms20.TileMatrixSet, tmIDs
 		levels = append(levels, level)
 	}
 
-	ix.InsertPolygon(polygon)
-	newPolygonsPerLevel := addPointsAndSnap(ix, polygon, levels, keepPointsAndLines)
+	err = ix.InsertPolygon(polygon)
+	if err != nil {
+		outsideGridErr := new(pointindex.OutsideGridError)
+		if errors.As(err, outsideGridErr) && config.IgnoreOutsideGrid {
+			log.Println("[WARNING] skipping polygon because: " + err.Error())
+			return make(map[tms20.TMID][]geom.Polygon)
+		} else {
+			panic(err)
+		}
+	}
+
+	newPolygonsPerLevel := addPointsAndSnap(ix, polygon, levels, config.KeepPointsAndLines)
 
 	newPolygonsPerTileMatrixID := make(map[tms20.TMID][]geom.Polygon, len(newPolygonsPerLevel))
 	for level, newPolygons := range newPolygonsPerLevel {