Merge pull request #71 from PDOK/cursor-gaps

Cursor gaps

.github/workflows/lint-go.yml

@@ -17,7 +17,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
           cache: false
 
       - uses: actions/checkout@v3
@@ -26,7 +26,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.52
+          version: latest
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

.github/workflows/tidy.yml

@@ -14,6 +14,6 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21'
       - uses: actions/checkout@v3
       - uses: katexochen/go-tidy-check@v2

Dockerfile

@@ -9,7 +9,7 @@ RUN npm install
 RUN npm run build
 
 ####### Go build
-FROM ${REGISTRY}/golang:1.20-bookworm AS build-env
+FROM ${REGISTRY}/golang:1.21-bookworm AS build-env
 WORKDIR /go/src/service
 ADD . /go/src/service
 

engine/config.go

@@ -273,9 +273,13 @@ type OgcAPIProcesses struct {
 }
 
 type Datasource struct {
-	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=FakeDB"`
-	FakeDB     bool        `yaml:"fakedb" validate:"required_without_all=GeoPackage"`
-	// Add more datasources here such as PostGIS, Mongo, Elastic, etc
+	GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=PostGIS"`
+	PostGIS    *PostGIS    `yaml:"postgis" validate:"required_without_all=GeoPackage"`
+	// Add more datasources here such as Mongo, Elastic, etc
+}
+
+type PostGIS struct {
+	// placeholder
 }
 
 type GeoPackage struct {

engine/contentnegotiation.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	formatParam   = "f"
+	FormatParam   = "f"
 	languageParam = "lang"
 
 	MediaTypeJSON          = "application/json"
@@ -127,11 +127,11 @@ func (cn *ContentNegotiation) formatToMediaType(format string) string {
 func (cn *ContentNegotiation) getFormatFromQueryParam(req *http.Request) string {
 	var requestedFormat = ""
 	queryParams := req.URL.Query()
-	if queryParams.Get(formatParam) != "" {
-		requestedFormat = queryParams.Get(formatParam)
+	if queryParams.Get(FormatParam) != "" {
+		requestedFormat = queryParams.Get(FormatParam)
 
 		// remove ?f= parameter, to prepare for rewrite
-		queryParams.Del(formatParam)
+		queryParams.Del(FormatParam)
 		req.URL.RawQuery = queryParams.Encode()
 	}
 	return requestedFormat

engine/openapi.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	texttemplate "text/template"
 
+	"github.com/PDOK/gokoala/engine/util"
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/getkin/kin-openapi/routers"
@@ -80,7 +81,7 @@ func newOpenAPI(config *Config, openAPIFile string) *OpenAPI {
 	return &OpenAPI{
 		config:   config,
 		spec:     resultSpec,
-		SpecJSON: prettyPrintJSON(resultSpecJSON, ""),
+		SpecJSON: util.PrettyPrintJSON(resultSpecJSON, ""),
 		router:   newOpenAPIRouter(resultSpec),
 	}
 }
@@ -140,7 +141,7 @@ func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3.
 			mergedJSON = specJSON
 		} else {
 			var err error
-			mergedJSON, err = mergeJSON(resultSpecJSON, specJSON)
+			mergedJSON, err = util.MergeJSON(resultSpecJSON, specJSON)
 			if err != nil {
 				log.Print(string(mergedJSON))
 				log.Fatalf("failed to merge openapi specs: %v", err)

engine/template.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	texttemplate "text/template"
 
+	"github.com/PDOK/gokoala/engine/util"
 	sprig "github.com/go-task/slim-sprig"
 	gomarkdown "github.com/gomarkdown/markdown"
 	gomarkdownhtml "github.com/gomarkdown/markdown/html"
@@ -212,7 +213,7 @@ func (t *Templates) renderNonHTMLTemplate(parsed *texttemplate.Template, params
 	var result = rendered.Bytes()
 	if strings.Contains(key.Format, FormatJSON) {
 		// pretty print all JSON (or derivatives like TileJSON)
-		result = prettyPrintJSON(result, key.Name)
+		result = util.PrettyPrintJSON(result, key.Name)
 	}
 	return result
 }

engine/json.go → engine/util/json.go

@@ -1,12 +1,12 @@
-package engine
+package util
 
 import (
 	"bytes"
 	"encoding/json"
 	"log"
 )
 
-func prettyPrintJSON(content []byte, name string) []byte {
+func PrettyPrintJSON(content []byte, name string) []byte {
 	var pretty bytes.Buffer
 	if err := json.Indent(&pretty, content, "", " "); err != nil {
 		log.Print(string(content))
@@ -15,14 +15,14 @@ func prettyPrintJSON(content []byte, name string) []byte {
 	return pretty.Bytes()
 }
 
-// mergeJSON merges the two JSON byte slices containing x1 and x2,
+// MergeJSON merges the two JSON byte slices containing x1 and x2,
 // preferring x1 over x2 except where x1 and x2 are
 // JSON objects, in which case the keys from both objects
 // are included and their values merged recursively.
 //
 // It returns an error if x1 or x2 cannot be JSON-unmarshalled,
 // or the merged JSON is invalid.
-func mergeJSON(x1, x2 []byte) ([]byte, error) {
+func MergeJSON(x1, x2 []byte) ([]byte, error) {
 	var j1 interface{}
 	err := json.Unmarshal(x1, &j1)
 	if err != nil {

engine/json_test.go → engine/util/json_test.go

@@ -1,4 +1,4 @@
-package engine
+package util
 
 import (
 	"encoding/json"
@@ -11,7 +11,7 @@ import (
 
 func TestJSONMerge_identical_json_input_should_not_result_differences(t *testing.T) {
 	// given
-	file, err := filepath.Abs("engine/testdata/ogcapi-tiles-1.bundled.json")
+	file, err := filepath.Abs("../testdata/ogcapi-tiles-1.bundled.json")
 	if err != nil {
 		t.Fatalf("can't locate testdata %v", err)
 	}
@@ -30,7 +30,7 @@ func TestJSONMerge_identical_json_input_should_not_result_differences(t *testing
 	}
 
 	// when
-	actual, err := mergeJSON(fileContent, fileContent)
+	actual, err := MergeJSON(fileContent, fileContent)
 	if err != nil {
 		t.Fatalf("JSON merge failed %v", err)
 	}

engine/util/maps.go

@@ -0,0 +1,11 @@
+package util
+
+// Keys returns the keys of the map m.
+// The keys will be an indeterminate order.
+func Keys[M ~map[K]V, K comparable, V any](m M) []K {
+	r := make([]K, 0, len(m))
+	for k := range m {
+		r = append(r, k)
+	}
+	return r
+}

examples/README.md

@@ -13,12 +13,11 @@ This example uses vector tiles from the [PDOK BGT dataset](https://www.pdok.nl/i
 
 ## OGC API Features example
 
-There are 3 examples configurations:
-- `config_features_fake.yaml` - use an in-memory datasource with fake (generated) features
+There are 2 examples configurations:
 - `config_features_local.yaml` - use the local [addresses.gpkg](resources%2Faddresses.gpkg) geopackage
 - `config_features_azure.yaml` - use [addresses.gpkg](resources%2Faddresses.gpkg) hosted in Azure Blob as a [Cloud-Backed SQLite/Geopackage](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki).
 
-For the first two (fake and local) just start GoKoala as specified in the root [README](../README.md#run)
+For the local version just start GoKoala as specified in the root [README](../README.md#run)
 and provide the mentioned config file.
 
 For the Azure example we use a local Azurite emulator which contains the cloud-backed `addresses.gpkg`:

go.mod

@@ -1,11 +1,10 @@
 module github.com/PDOK/gokoala
 
-go 1.20
+go 1.21
 
 require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/PDOK/go-cloud-sqlite-vfs v0.2.4
-	github.com/brianvoe/gofakeit/v6 v6.23.2
 	github.com/elnormous/contenttype v1.0.4
 	github.com/getkin/kin-openapi v0.116.0
 	github.com/go-chi/chi/v5 v5.0.8
@@ -16,7 +15,6 @@ require (
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/nicksnyder/go-i18n/v2 v2.2.1
-	github.com/sqids/sqids-go v0.4.1
 	github.com/stretchr/testify v1.8.2
 	github.com/urfave/cli/v2 v2.25.3
 	github.com/writeas/go-strip-markdown/v2 v2.1.1

go.sum

@@ -4,8 +4,6 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
 github.com/PDOK/go-cloud-sqlite-vfs v0.2.4 h1:OMUbfVBcue/qmfInQEwCD56pRJg0TqtXUGESGLuqxPM=
 github.com/PDOK/go-cloud-sqlite-vfs v0.2.4/go.mod h1:+mZxO6New9AlVqFAF2rBEsOZB7J2aavwtdn3ifg021s=
 github.com/arolek/p v0.0.0-20191103215535-df3c295ed582/go.mod h1:JPNItmi3yb44Q5QWM+Kh5n9oeRhfcJzPNS90mbLo25U=
-github.com/brianvoe/gofakeit/v6 v6.23.2 h1:lVde18uhad5wII/f5RMVFLtdQNE0HaGFuBUXmYKk8i8=
-github.com/brianvoe/gofakeit/v6 v6.23.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -27,6 +25,7 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -86,8 +85,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sqids/sqids-go v0.4.1 h1:eQKYzmAZbLlRwHeHYPF35QhgxwZHLnlmVj9AkIj/rrw=
-github.com/sqids/sqids-go v0.4.1/go.mod h1:EMwHuPQgSNFS0A49jESTfIQS+066XQTVhukrzEPScl8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

ogc/features/datasources/datasource.go

@@ -10,8 +10,8 @@ import (
 // Datasource holding all the features for a single dataset
 type Datasource interface {
 
-	// GetFeatures returns a FeatureCollection from the underlying datasource and a Cursor for pagination
-	GetFeatures(ctx context.Context, collection string, params QueryParams) (*domain.FeatureCollection, domain.Cursor, error)
+	// GetFeatures returns a FeatureCollection from the underlying datasource and Cursors for pagination
+	GetFeatures(ctx context.Context, collection string, options FeatureOptions) (*domain.FeatureCollection, domain.Cursors, error)
 
 	// GetFeature returns a specific Feature from the FeatureCollection of the underlying datasource
 	GetFeature(ctx context.Context, collection string, featureID int64) (*domain.Feature, error)
@@ -20,10 +20,20 @@ type Datasource interface {
 	Close()
 }
 
-// QueryParams to select a certain set of Features
-type QueryParams struct {
-	Cursor  int64
-	Limit   int
+// FeatureOptions to select a certain set of Features
+type FeatureOptions struct {
+	// pagination
+	Cursor domain.DecodedCursor
+	Limit  int
+
+	// multiple projections support
+	Crs string
+
+	// filtering by bounding box
 	Bbox    *geom.Extent
 	BboxCrs string
+
+	// filtering by CQL
+	Filter    string
+	FilterCrs string
 }