diff --git a/.github/workflows/lint-go.yml b/.github/workflows/lint-go.yml index e773b9c8..ad3e9088 100644 --- a/.github/workflows/lint-go.yml +++ b/.github/workflows/lint-go.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' cache: false - uses: actions/checkout@v3 @@ -26,7 +26,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.52 + version: latest # Optional: working directory, useful for monorepos # working-directory: somedir diff --git a/.github/workflows/tidy.yml b/.github/workflows/tidy.yml index 589b8305..7ad13f9c 100644 --- a/.github/workflows/tidy.yml +++ b/.github/workflows/tidy.yml @@ -14,6 +14,6 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/checkout@v3 - uses: katexochen/go-tidy-check@v2 diff --git a/Dockerfile b/Dockerfile index ffb51b66..045e6116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN npm install RUN npm run build ####### Go build -FROM ${REGISTRY}/golang:1.20-bookworm AS build-env +FROM ${REGISTRY}/golang:1.21-bookworm AS build-env WORKDIR /go/src/service ADD . /go/src/service diff --git a/engine/config.go b/engine/config.go index 7b6948ca..db951820 100644 --- a/engine/config.go +++ b/engine/config.go @@ -273,9 +273,13 @@ type OgcAPIProcesses struct { } type Datasource struct { - GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=FakeDB"` - FakeDB bool `yaml:"fakedb" validate:"required_without_all=GeoPackage"` - // Add more datasources here such as PostGIS, Mongo, Elastic, etc + GeoPackage *GeoPackage `yaml:"geopackage" validate:"required_without_all=PostGIS"` + PostGIS *PostGIS `yaml:"postgis" validate:"required_without_all=GeoPackage"` + // Add more datasources here such as Mongo, Elastic, etc +} + +type PostGIS struct { + // placeholder } type GeoPackage struct { diff --git a/engine/contentnegotiation.go b/engine/contentnegotiation.go index 98747e48..a4183f70 100644 --- a/engine/contentnegotiation.go +++ b/engine/contentnegotiation.go @@ -9,7 +9,7 @@ import ( ) const ( - formatParam = "f" + FormatParam = "f" languageParam = "lang" MediaTypeJSON = "application/json" @@ -127,11 +127,11 @@ func (cn *ContentNegotiation) formatToMediaType(format string) string { func (cn *ContentNegotiation) getFormatFromQueryParam(req *http.Request) string { var requestedFormat = "" queryParams := req.URL.Query() - if queryParams.Get(formatParam) != "" { - requestedFormat = queryParams.Get(formatParam) + if queryParams.Get(FormatParam) != "" { + requestedFormat = queryParams.Get(FormatParam) // remove ?f= parameter, to prepare for rewrite - queryParams.Del(formatParam) + queryParams.Del(FormatParam) req.URL.RawQuery = queryParams.Encode() } return requestedFormat diff --git a/engine/openapi.go b/engine/openapi.go index 775b40e1..f1264730 100644 --- a/engine/openapi.go +++ b/engine/openapi.go @@ -15,6 +15,7 @@ import ( "strings" texttemplate "text/template" + "github.com/PDOK/gokoala/engine/util" "github.com/getkin/kin-openapi/openapi3" "github.com/getkin/kin-openapi/openapi3filter" "github.com/getkin/kin-openapi/routers" @@ -80,7 +81,7 @@ func newOpenAPI(config *Config, openAPIFile string) *OpenAPI { return &OpenAPI{ config: config, spec: resultSpec, - SpecJSON: prettyPrintJSON(resultSpecJSON, ""), + SpecJSON: util.PrettyPrintJSON(resultSpecJSON, ""), router: newOpenAPIRouter(resultSpec), } } @@ -140,7 +141,7 @@ func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3. mergedJSON = specJSON } else { var err error - mergedJSON, err = mergeJSON(resultSpecJSON, specJSON) + mergedJSON, err = util.MergeJSON(resultSpecJSON, specJSON) if err != nil { log.Print(string(mergedJSON)) log.Fatalf("failed to merge openapi specs: %v", err) diff --git a/engine/slices.go b/engine/slices.go deleted file mode 100644 index da22d64b..00000000 --- a/engine/slices.go +++ /dev/null @@ -1,31 +0,0 @@ -package engine - -// Contains reports whether v is present in s. -// -// Source: https://github.com/golang/exp/blob/master/slices/slices.go -func Contains[E comparable](s []E, v E) bool { - return Index(s, v) >= 0 -} - -// Index returns the index of the first occurrence of v in s, -// or -1 if not present. -// -// Source: https://github.com/golang/exp/blob/master/slices/slices.go -func Index[E comparable](s []E, v E) int { - for i, vs := range s { - if v == vs { - return i - } - } - return -1 -} - -// Keys returns the keys of the map m. -// The keys will be an indeterminate order. -func Keys[M ~map[K]V, K comparable, V any](m M) []K { - r := make([]K, 0, len(m)) - for k := range m { - r = append(r, k) - } - return r -} diff --git a/engine/template.go b/engine/template.go index 5171601e..c8738713 100644 --- a/engine/template.go +++ b/engine/template.go @@ -14,6 +14,7 @@ import ( "strings" texttemplate "text/template" + "github.com/PDOK/gokoala/engine/util" sprig "github.com/go-task/slim-sprig" gomarkdown "github.com/gomarkdown/markdown" gomarkdownhtml "github.com/gomarkdown/markdown/html" @@ -212,7 +213,7 @@ func (t *Templates) renderNonHTMLTemplate(parsed *texttemplate.Template, params var result = rendered.Bytes() if strings.Contains(key.Format, FormatJSON) { // pretty print all JSON (or derivatives like TileJSON) - result = prettyPrintJSON(result, key.Name) + result = util.PrettyPrintJSON(result, key.Name) } return result } diff --git a/engine/json.go b/engine/util/json.go similarity index 89% rename from engine/json.go rename to engine/util/json.go index 209fecc1..9faad947 100644 --- a/engine/json.go +++ b/engine/util/json.go @@ -1,4 +1,4 @@ -package engine +package util import ( "bytes" @@ -6,7 +6,7 @@ import ( "log" ) -func prettyPrintJSON(content []byte, name string) []byte { +func PrettyPrintJSON(content []byte, name string) []byte { var pretty bytes.Buffer if err := json.Indent(&pretty, content, "", " "); err != nil { log.Print(string(content)) @@ -15,14 +15,14 @@ func prettyPrintJSON(content []byte, name string) []byte { return pretty.Bytes() } -// mergeJSON merges the two JSON byte slices containing x1 and x2, +// MergeJSON merges the two JSON byte slices containing x1 and x2, // preferring x1 over x2 except where x1 and x2 are // JSON objects, in which case the keys from both objects // are included and their values merged recursively. // // It returns an error if x1 or x2 cannot be JSON-unmarshalled, // or the merged JSON is invalid. -func mergeJSON(x1, x2 []byte) ([]byte, error) { +func MergeJSON(x1, x2 []byte) ([]byte, error) { var j1 interface{} err := json.Unmarshal(x1, &j1) if err != nil { diff --git a/engine/json_test.go b/engine/util/json_test.go similarity index 84% rename from engine/json_test.go rename to engine/util/json_test.go index 5c5ec2b1..869b4b39 100644 --- a/engine/json_test.go +++ b/engine/util/json_test.go @@ -1,4 +1,4 @@ -package engine +package util import ( "encoding/json" @@ -11,7 +11,7 @@ import ( func TestJSONMerge_identical_json_input_should_not_result_differences(t *testing.T) { // given - file, err := filepath.Abs("engine/testdata/ogcapi-tiles-1.bundled.json") + file, err := filepath.Abs("../testdata/ogcapi-tiles-1.bundled.json") if err != nil { t.Fatalf("can't locate testdata %v", err) } @@ -30,7 +30,7 @@ func TestJSONMerge_identical_json_input_should_not_result_differences(t *testing } // when - actual, err := mergeJSON(fileContent, fileContent) + actual, err := MergeJSON(fileContent, fileContent) if err != nil { t.Fatalf("JSON merge failed %v", err) } diff --git a/engine/util/maps.go b/engine/util/maps.go new file mode 100644 index 00000000..63bdbab0 --- /dev/null +++ b/engine/util/maps.go @@ -0,0 +1,11 @@ +package util + +// Keys returns the keys of the map m. +// The keys will be an indeterminate order. +func Keys[M ~map[K]V, K comparable, V any](m M) []K { + r := make([]K, 0, len(m)) + for k := range m { + r = append(r, k) + } + return r +} diff --git a/examples/README.md b/examples/README.md index c1513cc0..e53b63ca 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,12 +13,11 @@ This example uses vector tiles from the [PDOK BGT dataset](https://www.pdok.nl/i ## OGC API Features example -There are 3 examples configurations: -- `config_features_fake.yaml` - use an in-memory datasource with fake (generated) features +There are 2 examples configurations: - `config_features_local.yaml` - use the local [addresses.gpkg](resources%2Faddresses.gpkg) geopackage - `config_features_azure.yaml` - use [addresses.gpkg](resources%2Faddresses.gpkg) hosted in Azure Blob as a [Cloud-Backed SQLite/Geopackage](https://sqlite.org/cloudsqlite/doc/trunk/www/index.wiki). -For the first two (fake and local) just start GoKoala as specified in the root [README](../README.md#run) +For the local version just start GoKoala as specified in the root [README](../README.md#run) and provide the mentioned config file. For the Azure example we use a local Azurite emulator which contains the cloud-backed `addresses.gpkg`: diff --git a/go.mod b/go.mod index 3f089bbc..357701de 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,10 @@ module github.com/PDOK/gokoala -go 1.20 +go 1.21 require ( github.com/BurntSushi/toml v1.3.2 github.com/PDOK/go-cloud-sqlite-vfs v0.2.4 - github.com/brianvoe/gofakeit/v6 v6.23.2 github.com/elnormous/contenttype v1.0.4 github.com/getkin/kin-openapi v0.116.0 github.com/go-chi/chi/v5 v5.0.8 @@ -16,7 +15,6 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/mattn/go-sqlite3 v1.14.17 github.com/nicksnyder/go-i18n/v2 v2.2.1 - github.com/sqids/sqids-go v0.4.1 github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.25.3 github.com/writeas/go-strip-markdown/v2 v2.1.1 diff --git a/go.sum b/go.sum index 1de8ed13..a9bf51cc 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/PDOK/go-cloud-sqlite-vfs v0.2.4 h1:OMUbfVBcue/qmfInQEwCD56pRJg0TqtXUGESGLuqxPM= github.com/PDOK/go-cloud-sqlite-vfs v0.2.4/go.mod h1:+mZxO6New9AlVqFAF2rBEsOZB7J2aavwtdn3ifg021s= github.com/arolek/p v0.0.0-20191103215535-df3c295ed582/go.mod h1:JPNItmi3yb44Q5QWM+Kh5n9oeRhfcJzPNS90mbLo25U= -github.com/brianvoe/gofakeit/v6 v6.23.2 h1:lVde18uhad5wII/f5RMVFLtdQNE0HaGFuBUXmYKk8i8= -github.com/brianvoe/gofakeit/v6 v6.23.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -27,6 +25,7 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -86,8 +85,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sqids/sqids-go v0.4.1 h1:eQKYzmAZbLlRwHeHYPF35QhgxwZHLnlmVj9AkIj/rrw= -github.com/sqids/sqids-go v0.4.1/go.mod h1:EMwHuPQgSNFS0A49jESTfIQS+066XQTVhukrzEPScl8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/ogc/features/datasources/datasource.go b/ogc/features/datasources/datasource.go index 06f47996..9fc517b5 100644 --- a/ogc/features/datasources/datasource.go +++ b/ogc/features/datasources/datasource.go @@ -10,8 +10,8 @@ import ( // Datasource holding all the features for a single dataset type Datasource interface { - // GetFeatures returns a FeatureCollection from the underlying datasource and a Cursor for pagination - GetFeatures(ctx context.Context, collection string, params QueryParams) (*domain.FeatureCollection, domain.Cursor, error) + // GetFeatures returns a FeatureCollection from the underlying datasource and Cursors for pagination + GetFeatures(ctx context.Context, collection string, options FeatureOptions) (*domain.FeatureCollection, domain.Cursors, error) // GetFeature returns a specific Feature from the FeatureCollection of the underlying datasource GetFeature(ctx context.Context, collection string, featureID int64) (*domain.Feature, error) @@ -20,10 +20,20 @@ type Datasource interface { Close() } -// QueryParams to select a certain set of Features -type QueryParams struct { - Cursor int64 - Limit int +// FeatureOptions to select a certain set of Features +type FeatureOptions struct { + // pagination + Cursor domain.DecodedCursor + Limit int + + // multiple projections support + Crs string + + // filtering by bounding box Bbox *geom.Extent BboxCrs string + + // filtering by CQL + Filter string + FilterCrs string } diff --git a/ogc/features/datasources/fakedb/fakedb.go b/ogc/features/datasources/fakedb/fakedb.go deleted file mode 100644 index 9b0b7dc6..00000000 --- a/ogc/features/datasources/fakedb/fakedb.go +++ /dev/null @@ -1,88 +0,0 @@ -package fakedb - -import ( - "context" - "sort" - - "github.com/PDOK/gokoala/ogc/features/datasources" - "github.com/PDOK/gokoala/ogc/features/domain" - "github.com/brianvoe/gofakeit/v6" - "github.com/go-spatial/geom" -) - -const nrOfFakeFeatures = 10000 - -// FakeDB fake/mock datasource used for prototyping/testing/demos/etc. -type FakeDB struct { - featureCollection *domain.FeatureCollection -} - -func NewFakeDB() *FakeDB { - return &FakeDB{ - featureCollection: generateFakeFeatureCollection(), - } -} - -func (FakeDB) Close() { - // noop -} - -func (fdb FakeDB) GetFeatures(_ context.Context, _ string, params datasources.QueryParams) (*domain.FeatureCollection, domain.Cursor, error) { - low := params.Cursor - high := low + int64(params.Limit) - - last := high > int64(len(fdb.featureCollection.Features)) - if last { - high = int64(len(fdb.featureCollection.Features)) - } - if high < 0 { - high = 0 - } - - page := fdb.featureCollection.Features[low:high] - return &domain.FeatureCollection{ - NumberReturned: len(page), - Features: page, - }, - domain.NewCursor(page, last), - nil -} - -func (fdb FakeDB) GetFeature(_ context.Context, _ string, featureID int64) (*domain.Feature, error) { - for _, feat := range fdb.featureCollection.Features { - if feat.ID == featureID { - return feat, nil - } - } - return nil, nil //nolint:nilnil -} - -func generateFakeFeatureCollection() *domain.FeatureCollection { - var feats []*domain.Feature - for i := 0; i < nrOfFakeFeatures; i++ { - address := gofakeit.Address() - var props = map[string]interface{}{ - "streetname": address.Street, - "city": address.City, - "year": gofakeit.Year(), - "floorsize": gofakeit.Number(10, 300), - "purpose": gofakeit.Blurb(), - } - - feature := domain.Feature{} - feature.ID = int64(i) - feature.Geometry.Geometry = geom.Point{address.Longitude, address.Latitude} - feature.Properties = props - - feats = append(feats, &feature) - } - - // the collection must be ordered by the cursor column - sort.Slice(feats, func(i, j int) bool { - return feats[i].ID < feats[j].ID - }) - - fc := domain.FeatureCollection{} - fc.Features = feats - return &fc -} diff --git a/ogc/features/datasources/geopackage/geopackage.go b/ogc/features/datasources/geopackage/geopackage.go index 775c4ae3..048aff7b 100644 --- a/ogc/features/datasources/geopackage/geopackage.go +++ b/ogc/features/datasources/geopackage/geopackage.go @@ -7,6 +7,7 @@ import ( "time" "github.com/PDOK/gokoala/engine" + "github.com/PDOK/gokoala/engine/util" "github.com/PDOK/gokoala/ogc/features/datasources" "github.com/PDOK/gokoala/ogc/features/domain" "github.com/go-spatial/geom" @@ -84,46 +85,48 @@ func (g *GeoPackage) Close() { g.backend.close() } -func (g *GeoPackage) GetFeatures(ctx context.Context, collection string, params datasources.QueryParams) (*domain.FeatureCollection, domain.Cursor, error) { +func (g *GeoPackage) GetFeatures(ctx context.Context, collection string, options datasources.FeatureOptions) (*domain.FeatureCollection, domain.Cursors, error) { table, ok := g.featureTableByCollectionID[collection] if !ok { - return nil, domain.Cursor{}, fmt.Errorf("can't query collection '%s' since it doesn't exist in "+ - "geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByCollectionID)) + return nil, domain.Cursors{}, fmt.Errorf("can't query collection '%s' since it doesn't exist in "+ + "geopackage, available in geopackage: %v", collection, util.Keys(g.featureTableByCollectionID)) } queryCtx, cancel := context.WithTimeout(ctx, g.queryTimeout) // https://go.dev/doc/database/cancel-operations defer cancel() - query, queryArgs := g.makeFeaturesQuery(table, params) + query, queryArgs := g.makeFeaturesQuery(table, options) stmt, err := g.backend.getDB().PreparexContext(ctx, query) if err != nil { - return nil, domain.Cursor{}, err + return nil, domain.Cursors{}, err } defer stmt.Close() rows, err := g.backend.getDB().QueryxContext(queryCtx, query, queryArgs...) if err != nil { - return nil, domain.Cursor{}, fmt.Errorf("query '%s' failed: %w", query, err) + return nil, domain.Cursors{}, fmt.Errorf("query '%s' failed: %w", query, err) } defer rows.Close() + var nextPrev *domain.PrevNextFID result := domain.FeatureCollection{} - result.Features, err = domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry) + result.Features, nextPrev, err = domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry) if err != nil { - return nil, domain.Cursor{}, err + return nil, domain.Cursors{}, err + } + if nextPrev == nil { + return nil, domain.Cursors{}, fmt.Errorf("failed to get prev/next cursor") } result.NumberReturned = len(result.Features) - last := result.NumberReturned < params.Limit // we could make this more reliable (by querying one record more), but sufficient for now - - return &result, domain.NewCursor(result.Features, last), nil + return &result, domain.NewCursors(*nextPrev, options.Cursor.FiltersChecksum), nil } func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureID int64) (*domain.Feature, error) { table, ok := g.featureTableByCollectionID[collection] if !ok { return nil, fmt.Errorf("can't query collection '%s' since it doesn't exist in "+ - "geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByCollectionID)) + "geopackage, available in geopackage: %v", collection, util.Keys(g.featureTableByCollectionID)) } queryCtx, cancel := context.WithTimeout(ctx, g.queryTimeout) // https://go.dev/doc/database/cancel-operations @@ -142,7 +145,7 @@ func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureI } defer rows.Close() - features, err := domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry) + features, _, err := domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry) if err != nil { return nil, err } @@ -152,14 +155,30 @@ func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureI return features[0], nil } -func (g *GeoPackage) makeFeaturesQuery(table *featureTable, params datasources.QueryParams) (string, []any) { - if params.Bbox != nil { +func (g *GeoPackage) makeFeaturesQuery(table *featureTable, opt datasources.FeatureOptions) (string, []any) { + // filters (bbox, part3 filters) need to be included in the next/prev handling as well + extraFilters := "" + + // make sure to use SQL bind variables, we prefer $N style parameters for reusability and broader compatibility. + nextPrevCTE := fmt.Sprintf(` +next as (select * from %[1]s where %[2]s >= $1 %[3]s order by %[2]s asc limit $2 + 1), +prev as (select * from %[1]s where %[2]s < $1 %[3]s order by %[2]s desc limit $2), +nextprev as (select * from next union all select * from prev), +featuretable as (select *, lag(%[2]s, $2) over (order by %[2]s) as prevfid, lead(%[2]s, $2) over (order by %[2]s) as nextfid from nextprev) +`, table.TableName, g.fidColumn, extraFilters) + + if opt.Bbox != nil { // TODO create bbox query - bboxQuery := "" - return bboxQuery, []any{params.Cursor, params.Limit, params.Bbox} + bboxQuery := fmt.Sprintf(`with %s `, nextPrevCTE) + return bboxQuery, []any{opt.Cursor.FID, opt.Limit, opt.Bbox} } - defaultQuery := fmt.Sprintf("select * from %s f where f.%s > ? order by f.%s limit ?", table.TableName, g.fidColumn, g.fidColumn) - return defaultQuery, []any{params.Cursor, params.Limit} + if opt.Filter != "" { + // TODO create part3 filter query + filterQuery := fmt.Sprintf(`with %s `, nextPrevCTE) + return filterQuery, []any{opt.Cursor.FID, opt.Limit, opt.Filter} + } + defaultQuery := fmt.Sprintf(`with %s select * from featuretable where %s >= $1 limit $2`, nextPrevCTE, g.fidColumn) + return defaultQuery, []any{opt.Cursor.FID, opt.Limit} } // Read gpkg_contents table. This table contains metadata about feature tables. The result is a mapping from @@ -190,7 +209,7 @@ func readGpkgContents(collections engine.GeoSpatialCollections, db *sqlx.DB) (ma if row.Identifier == collection.ID { result[collection.ID] = &row break - } else if collection.Features.DatasourceID != nil && row.Identifier == *collection.Features.DatasourceID { + } else if hasMatchingDatasourceID(collection, row) { result[collection.ID] = &row break } @@ -208,6 +227,11 @@ func readGpkgContents(collections engine.GeoSpatialCollections, db *sqlx.DB) (ma return result, nil } +func hasMatchingDatasourceID(collection engine.GeoSpatialCollection, row featureTable) bool { + return collection.Features != nil && collection.Features.DatasourceID != nil && + row.Identifier == *collection.Features.DatasourceID +} + func readGpkgGeometry(rawGeom []byte) (geom.Geometry, error) { geometry, err := gpkg.DecodeGeometry(rawGeom) if err != nil { diff --git a/ogc/features/datasources/geopackage/geopackage_test.go b/ogc/features/datasources/geopackage/geopackage_test.go index 21907eb0..347ea7f9 100644 --- a/ogc/features/datasources/geopackage/geopackage_test.go +++ b/ogc/features/datasources/geopackage/geopackage_test.go @@ -68,14 +68,14 @@ func TestGeoPackage_GetFeatures(t *testing.T) { type args struct { ctx context.Context collection string - queryParams datasources.QueryParams + queryParams datasources.FeatureOptions } tests := []struct { name string fields fields args args wantFC *domain.FeatureCollection - wantCursor domain.Cursor + wantCursor domain.Cursors wantErr bool }{ { @@ -84,13 +84,13 @@ func TestGeoPackage_GetFeatures(t *testing.T) { backend: newAddressesGeoPackage(), fidColumn: "feature_id", featureTableByID: map[string]*featureTable{"ligplaatsen": {TableName: "ligplaatsen", GeometryColumnName: "geom"}}, - queryTimeout: 5 * time.Second, + queryTimeout: 60 * time.Second, }, args: args{ ctx: context.Background(), collection: "ligplaatsen", - queryParams: datasources.QueryParams{ - Cursor: 0, + queryParams: datasources.FeatureOptions{ + Cursor: domain.DecodedCursor{FID: 0, FiltersChecksum: []byte{}}, Limit: 2, }, }, @@ -115,9 +115,9 @@ func TestGeoPackage_GetFeatures(t *testing.T) { }, }, }, - wantCursor: domain.Cursor{ - Prev: "spDyEwb4", - Next: "trrEb5db", // 3837 + wantCursor: domain.Cursors{ + Prev: "fA==", + Next: "Dv58", // 3838 }, wantErr: false, }, @@ -132,9 +132,12 @@ func TestGeoPackage_GetFeatures(t *testing.T) { args: args{ ctx: context.Background(), collection: "ligplaatsen", - queryParams: datasources.QueryParams{ - Cursor: 3837, // see next cursor from test above - Limit: 3, + queryParams: datasources.FeatureOptions{ + Cursor: domain.DecodedCursor{ + FID: 3838, // see next cursor from test above + FiltersChecksum: []byte{}, + }, + Limit: 3, }, }, wantFC: &domain.FeatureCollection{ @@ -166,9 +169,9 @@ func TestGeoPackage_GetFeatures(t *testing.T) { }, }, }, - wantCursor: domain.Cursor{ - Prev: "LZZS4c3w", - Next: "CNNniQpu", + wantCursor: domain.Cursors{ + Prev: "fA==", + Next: "DwF8", }, wantErr: false, }, @@ -183,13 +186,13 @@ func TestGeoPackage_GetFeatures(t *testing.T) { args: args{ ctx: context.Background(), collection: "vakantiehuizen", // not in gpkg - queryParams: datasources.QueryParams{ - Cursor: 0, + queryParams: datasources.FeatureOptions{ + Cursor: domain.DecodedCursor{FID: 0, FiltersChecksum: []byte{}}, Limit: 10, }, }, wantFC: nil, - wantCursor: domain.Cursor{}, + wantCursor: domain.Cursors{}, wantErr: true, // should fail }, } diff --git a/ogc/features/datasources/postgis/postgis.go b/ogc/features/datasources/postgis/postgis.go new file mode 100644 index 00000000..855ae4ec --- /dev/null +++ b/ogc/features/datasources/postgis/postgis.go @@ -0,0 +1,33 @@ +package postgis + +import ( + "context" + "log" + + "github.com/PDOK/gokoala/ogc/features/datasources" + "github.com/PDOK/gokoala/ogc/features/domain" +) + +// PostGIS !!! Placeholder implementation, for future reference !!! +type PostGIS struct { +} + +func NewPostGIS() *PostGIS { + return &PostGIS{} +} + +func (PostGIS) Close() { + // noop +} + +func (pg PostGIS) GetFeatures(_ context.Context, _ string, _ datasources.FeatureOptions) (*domain.FeatureCollection, domain.Cursors, error) { + log.Fatal("PostGIS support is not implemented yet, this just serves to demonstrate that we can support multiple datastores") + return &domain.FeatureCollection{}, + domain.Cursors{}, + nil +} + +func (pg PostGIS) GetFeature(_ context.Context, _ string, _ int64) (*domain.Feature, error) { + log.Fatal("PostGIS support is not implemented yet, this just serves to demonstrate that we can support multiple datastores") + return nil, nil //nolint:nilnil +} diff --git a/ogc/features/domain/cursor.go b/ogc/features/domain/cursor.go new file mode 100644 index 00000000..7b29884f --- /dev/null +++ b/ogc/features/domain/cursor.go @@ -0,0 +1,101 @@ +package domain + +import ( + "bytes" + "encoding/base64" + "log" + "math/big" +) + +const separator = '|' + +// Cursors holds next and previous cursor. Note that we use +// 'cursor-based pagination' as opposed to 'offset-based pagination' +type Cursors struct { + Prev EncodedCursor + Next EncodedCursor + + HasPrev bool + HasNext bool +} + +// EncodedCursor is a scrambled string representation of the fields defined in DecodedCursor +type EncodedCursor string + +// DecodedCursor the cursor values after decoding EncodedCursor +type DecodedCursor struct { + FID int64 + FiltersChecksum []byte +} + +// PrevNextFID previous and next feature id (fid) to encode in cursor. +type PrevNextFID struct { + Prev int64 + Next int64 +} + +// NewCursors create Cursors based on the prev/next feature ids from the datasource +// and the provided filters (captured in a hash). +func NewCursors(fid PrevNextFID, filtersChecksum []byte) Cursors { + return Cursors{ + Prev: encodeCursor(fid.Prev, filtersChecksum), + Next: encodeCursor(fid.Next, filtersChecksum), + + HasPrev: fid.Prev > 0, + HasNext: fid.Next > 0, + } +} + +func encodeCursor(fid int64, filtersChecksum []byte) EncodedCursor { + fidAsBytes := big.NewInt(fid).Bytes() + + // format of the cursor: + cursor := fidAsBytes + cursor = append(cursor, byte(separator)) + cursor = append(cursor, filtersChecksum...) + + return EncodedCursor(base64.URLEncoding.EncodeToString(cursor)) +} + +// Decode turns encoded cursor into DecodedCursor and verifies the +// that the checksum of query params that act as filters hasn't changed +func (c EncodedCursor) Decode(filtersChecksum []byte) DecodedCursor { + value := string(c) + if value == "" { + return DecodedCursor{0, filtersChecksum} + } + + decoded, err := base64.URLEncoding.DecodeString(value) + if err != nil || len(decoded) == 0 { + log.Printf("decoding cursor value '%v' failed, defaulting to first page", decoded) + return DecodedCursor{0, filtersChecksum} + } + + decodedParts := bytes.Split(decoded, []byte{separator}) + if len(decoded) < 1 { + return DecodedCursor{0, filtersChecksum} + } + + // feature id + fid := big.NewInt(0).SetBytes(decodedParts[0]).Int64() + if err != nil { + log.Printf("cursor %s doesn't contain numeric value, defaulting to first page", decodedParts[0]) + return DecodedCursor{0, filtersChecksum} + } + if fid < 0 { + log.Printf("negative feature ID detected: %d, defaulting to first page", fid) + fid = 0 + } + + // checksum + if len(decodedParts) > 1 && !bytes.Equal(decodedParts[1], filtersChecksum) { + log.Printf("filters in query params have changed during pagination, resetting to first page") + return DecodedCursor{0, filtersChecksum} + } + + return DecodedCursor{fid, filtersChecksum} +} + +func (c EncodedCursor) String() string { + return string(c) +} diff --git a/ogc/features/domain/cursor_test.go b/ogc/features/domain/cursor_test.go new file mode 100644 index 00000000..54697ac4 --- /dev/null +++ b/ogc/features/domain/cursor_test.go @@ -0,0 +1,75 @@ +package domain + +import ( + "reflect" + "testing" +) + +func TestNewCursor(t *testing.T) { + type args struct { + features []*Feature + id PrevNextFID + } + var tests = []struct { + name string + args args + want Cursors + }{ + { + name: "test first page", + args: args{ + features: []*Feature{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}, + id: PrevNextFID{ + Prev: 0, + Next: 4, + }, + }, + want: Cursors{ + Prev: "fA==", + Next: "BHw=", + HasPrev: false, + HasNext: true, + }, + }, + { + name: "test last page", + args: args{ + features: []*Feature{{ID: 5}, {ID: 6}, {ID: 7}, {ID: 8}}, + id: PrevNextFID{ + Prev: 4, + Next: 0, + }, + }, + want: Cursors{ + Prev: "BHw=", + Next: "fA==", + HasPrev: true, + HasNext: false, + }, + }, + { + name: "test middle page", + args: args{ + features: []*Feature{{ID: 3}, {ID: 4}, {ID: 5}, {ID: 6}}, + id: PrevNextFID{ + Prev: 2, + Next: 7, + }, + }, + want: Cursors{ + Prev: "Anw=", + Next: "B3w=", + HasPrev: true, + HasNext: true, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewCursors(tt.args.id, []byte{}) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewCursors() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/ogc/features/domain/domain.go b/ogc/features/domain/domain.go deleted file mode 100644 index 9fa2c287..00000000 --- a/ogc/features/domain/domain.go +++ /dev/null @@ -1,128 +0,0 @@ -package domain - -import ( - "log" - "strconv" - - "github.com/go-spatial/geom/encoding/geojson" - "github.com/sqids/sqids-go" -) - -const ( - cursorAlphabet = "1Vti5BYcjOdTXunDozKPm4syvG6galxLM8eIrUS2bWqZCNkwpR309JFAHfh7EQ" // generated on https://sqids.org/playground -) - -var ( - cursorCodec, _ = sqids.New(sqids.Options{ - Alphabet: cursorAlphabet, - Blocklist: nil, // disable blocklist - MinLength: 8, - }) -) - -// featureCollectionType allows the GeoJSON type to be automatically set during json marshalling -type featureCollectionType struct{} - -func (fc *featureCollectionType) MarshalJSON() ([]byte, error) { - return []byte(`"FeatureCollection"`), nil -} - -func (fc *featureCollectionType) UnmarshalJSON([]byte) error { return nil } - -// FeatureCollection is a GeoJSON FeatureCollection with extras such as links -type FeatureCollection struct { - Links []Link `json:"links,omitempty"` - - NumberReturned int `json:"numberReturned"` - Type featureCollectionType `json:"type"` - Features []*Feature `json:"features"` -} - -// Feature is a GeoJSON Feature with extras such as links -type Feature struct { - // we overwrite ID since we want to make it a required attribute. We also expect feature ids to be - // auto-incrementing integers (which is the default in geopackages) since we use it for cursor-based pagination. - ID int64 `json:"id"` - Links []Link `json:"links,omitempty"` - - geojson.Feature -} - -// Link according to RFC 8288, https://datatracker.ietf.org/doc/html/rfc8288 -type Link struct { - Length int64 `json:"length,omitempty"` - Rel string `json:"rel"` - Title string `json:"title,omitempty"` - Type string `json:"type,omitempty"` - Href string `json:"href"` - Hreflang string `json:"hreflang,omitempty"` - Templated bool `json:"templated,omitempty"` -} - -// Cursor since we use cursor-based pagination as opposed to offset-based pagination -type Cursor struct { - Prev EncodedCursor - Next EncodedCursor - - IsFirst bool - IsLast bool -} - -func NewCursor(features []*Feature, last bool) Cursor { - limit := len(features) - if limit == 0 { - return Cursor{} - } - - start := features[0].ID - end := features[limit-1].ID - - prev := start - if prev != 0 { - prev -= int64(limit + 1) - if prev < 0 { - prev = 0 - } - } - next := end - - return Cursor{ - Prev: encodeCursor(prev), - Next: encodeCursor(next), - - IsFirst: next <= int64(limit), - IsLast: last, - } -} - -// EncodedCursor is a scrambled string representation of a consecutive ordered integer cursor -type EncodedCursor string - -func encodeCursor(value int64) EncodedCursor { - encodedValue, err := cursorCodec.Encode([]uint64{uint64(value)}) - if err != nil { - log.Printf("failed to encode cursor value %d, defaulting to unencoded value.", value) - return EncodedCursor(strconv.FormatInt(value, 10)) - } - return EncodedCursor(encodedValue) -} - -func (c EncodedCursor) Decode() int64 { - value := string(c) - if value == "" { - return 0 - } - decodedValue := cursorCodec.Decode(value) - if len(decodedValue) > 1 { - log.Printf("encountered more than one cursor value after decoding: '%v', "+ - "this is not allowed! Defaulting to first value.", decodedValue) - } else if len(decodedValue) == 0 { - log.Printf("decoding cursor value '%v' failed, defaulting to first page", decodedValue) - return 0 - } - result := int64(decodedValue[0]) - if result < 0 { - result = 0 - } - return result -} diff --git a/ogc/features/domain/domain_test.go b/ogc/features/domain/domain_test.go deleted file mode 100644 index ff43361d..00000000 --- a/ogc/features/domain/domain_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package domain - -import ( - "log" - "reflect" - "testing" -) - -func TestNewCursor(t *testing.T) { - type args struct { - features []*Feature - last bool - } - tests := []struct { - name string - args args - want Cursor - }{ - { - name: "test first page", - args: args{ - features: []*Feature{{ID: 1}, {ID: 2}, {ID: 3}, {ID: 4}}, - last: false, - }, - want: Cursor{ - Prev: "1GpOCgaM", - Next: "eVc7GU6Q", - IsFirst: true, - IsLast: false, - }, - }, - { - name: "test last page", - args: args{ - features: []*Feature{{ID: 5}, {ID: 6}, {ID: 7}, {ID: 8}}, - last: true, - }, - want: Cursor{ - Prev: "1GpOCgaM", - Next: "VCHYvtZJ", - IsFirst: false, - IsLast: true, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NewCursor(tt.args.features, tt.args.last) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("NewCursor() = %v, want %v", got, tt.want) - } - log.Printf("prev %d, next %d", got.Prev.Decode(), got.Next.Decode()) - }) - } -} diff --git a/ogc/features/domain/features.go b/ogc/features/domain/features.go new file mode 100644 index 00000000..86e687b4 --- /dev/null +++ b/ogc/features/domain/features.go @@ -0,0 +1,147 @@ +package domain + +import ( + "fmt" + "time" + + "github.com/go-spatial/geom" + "github.com/go-spatial/geom/encoding/geojson" + "github.com/jmoiron/sqlx" +) + +// featureCollectionType allows the GeoJSON type to be automatically set during json marshalling +type featureCollectionType struct{} + +func (fc *featureCollectionType) MarshalJSON() ([]byte, error) { + return []byte(`"FeatureCollection"`), nil +} + +func (fc *featureCollectionType) UnmarshalJSON([]byte) error { return nil } + +// FeatureCollection is a GeoJSON FeatureCollection with extras such as links +type FeatureCollection struct { + Links []Link `json:"links,omitempty"` + + NumberReturned int `json:"numberReturned"` + Type featureCollectionType `json:"type"` + Features []*Feature `json:"features"` +} + +// Feature is a GeoJSON Feature with extras such as links +type Feature struct { + // we overwrite ID since we want to make it a required attribute. We also expect feature ids to be + // auto-incrementing integers (which is the default in geopackages) since we use it for cursor-based pagination. + ID int64 `json:"id"` + Links []Link `json:"links,omitempty"` + + geojson.Feature +} + +// Link according to RFC 8288, https://datatracker.ietf.org/doc/html/rfc8288 +type Link struct { + Length int64 `json:"length,omitempty"` + Rel string `json:"rel"` + Title string `json:"title,omitempty"` + Type string `json:"type,omitempty"` + Href string `json:"href"` + Hreflang string `json:"hreflang,omitempty"` + Templated bool `json:"templated,omitempty"` +} + +// MapRowsToFeatures datasource agnostic mapper from SQL rows/result set to Features domain model +func MapRowsToFeatures(rows *sqlx.Rows, fidColumn string, geomColumn string, + geomMapper func([]byte) (geom.Geometry, error)) ([]*Feature, *PrevNextFID, error) { + + result := make([]*Feature, 0) + columns, err := rows.Columns() + if err != nil { + return result, nil, err + } + + firstRow := true + var nextPrevID *PrevNextFID + for rows.Next() { + var values []interface{} + if values, err = rows.SliceScan(); err != nil { + return result, nil, err + } + + feature := &Feature{Feature: geojson.Feature{Properties: make(map[string]interface{})}} + np, err := mapColumnsToFeature(firstRow, feature, columns, values, fidColumn, geomColumn, geomMapper) + if err != nil { + return result, nil, err + } else if firstRow { + nextPrevID = np + firstRow = false + } + result = append(result, feature) + } + return result, nextPrevID, nil +} + +//nolint:cyclop,funlen +func mapColumnsToFeature(firstRow bool, feature *Feature, columns []string, values []interface{}, + fidColumn string, geomColumn string, geomMapper func([]byte) (geom.Geometry, error)) (*PrevNextFID, error) { + + nextPrevID := PrevNextFID{} + for i, columnName := range columns { + columnValue := values[i] + if columnValue == nil { + continue + } + + switch columnName { + case fidColumn: + feature.ID = columnValue.(int64) + + case geomColumn: + rawGeom, ok := columnValue.([]byte) + if !ok { + return nil, fmt.Errorf("failed to read geometry from %s column in datasource", geomColumn) + } + mappedGeom, err := geomMapper(rawGeom) + if err != nil { + return nil, fmt.Errorf("failed to map/decode geometry from datasource, error: %w", err) + } + feature.Geometry = geojson.Geometry{Geometry: mappedGeom} + + case "minx", "miny", "maxx", "maxy", "min_zoom", "max_zoom": + // Skip these columns used for bounding box and zoom filtering + continue + + case "prevfid": + // Only the first row in the result set contains the previous feature id + if firstRow { + nextPrevID.Prev = columnValue.(int64) + } + + case "nextfid": + // Only the first row in the result set contains the next feature id + if firstRow { + nextPrevID.Next = columnValue.(int64) + } + + default: + // Grab any non-nil, non-id, non-bounding box, & non-geometry column as a tag + switch v := columnValue.(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + feature.Properties[columnName] = string(asBytes) + case int64: + feature.Properties[columnName] = v + case float64: + feature.Properties[columnName] = v + case time.Time: + feature.Properties[columnName] = v + case string: + feature.Properties[columnName] = v + case bool: + feature.Properties[columnName] = v + default: + return nil, fmt.Errorf("unexpected type for sqlite column data: %v: %T", columns[i], v) + } + } + } + return &nextPrevID, nil +} diff --git a/ogc/features/domain/mapper.go b/ogc/features/domain/mapper.go deleted file mode 100644 index 14ed6b1c..00000000 --- a/ogc/features/domain/mapper.go +++ /dev/null @@ -1,89 +0,0 @@ -package domain - -import ( - "fmt" - "time" - - "github.com/go-spatial/geom" - "github.com/go-spatial/geom/encoding/geojson" - "github.com/jmoiron/sqlx" -) - -// MapRowsToFeatures datasource agnostic mapper from SQL rows/result set to Features domain model -func MapRowsToFeatures(rows *sqlx.Rows, fidColumn string, geomColumn string, - geomMapper func([]byte) (geom.Geometry, error)) ([]*Feature, error) { - - result := make([]*Feature, 0) - columns, err := rows.Columns() - if err != nil { - return result, err - } - - for rows.Next() { - var values []interface{} - if values, err = rows.SliceScan(); err != nil { - return result, err - } - feature := &Feature{Feature: geojson.Feature{Properties: make(map[string]interface{})}} - - if err = mapColumnsToFeature(feature, columns, values, fidColumn, geomColumn, geomMapper); err != nil { - return result, err - } - result = append(result, feature) - } - return result, nil -} - -//nolint:cyclop -func mapColumnsToFeature(feature *Feature, columns []string, values []interface{}, fidColumn string, - geomColumn string, geomMapper func([]byte) (geom.Geometry, error)) error { - - for i, columnName := range columns { - columnValue := values[i] - if columnValue == nil { - continue - } - - switch columnName { - case fidColumn: - feature.ID = columnValue.(int64) - - case geomColumn: - rawGeom, ok := columnValue.([]byte) - if !ok { - return fmt.Errorf("failed to read geometry from %s column in datasource", geomColumn) - } - mappedGeom, err := geomMapper(rawGeom) - if err != nil { - return fmt.Errorf("failed to map/decode geometry from datasource, error: %w", err) - } - feature.Geometry = geojson.Geometry{Geometry: mappedGeom} - - case "minx", "miny", "maxx", "maxy", "min_zoom", "max_zoom": - // Skip these columns used for bounding box and zoom filtering - continue - - default: - // Grab any non-nil, non-id, non-bounding box, & non-geometry column as a tag - switch v := columnValue.(type) { - case []uint8: - asBytes := make([]byte, len(v)) - copy(asBytes, v) - feature.Properties[columnName] = string(asBytes) - case int64: - feature.Properties[columnName] = v - case float64: - feature.Properties[columnName] = v - case time.Time: - feature.Properties[columnName] = v - case string: - feature.Properties[columnName] = v - case bool: - feature.Properties[columnName] = v - default: - return fmt.Errorf("unexpected type for sqlite column data: %v: %T", columns[i], v) - } - } - } - return nil -} diff --git a/ogc/features/html.go b/ogc/features/html.go index f8275f7e..b668e3a7 100644 --- a/ogc/features/html.go +++ b/ogc/features/html.go @@ -8,6 +8,10 @@ import ( "github.com/PDOK/gokoala/ogc/features/domain" ) +const ( + collectionsCrumb = "collections/" +) + var ( collectionsBreadcrumb = []engine.Breadcrumb{ { @@ -38,7 +42,9 @@ type featureCollectionPage struct { CollectionID string Metadata *engine.GeoSpatialCollectionMetadata - Cursor domain.Cursor + Cursor domain.Cursors + PrevLink string + NextLink string Limit int } @@ -51,7 +57,7 @@ type featurePage struct { } func (hf *htmlFeatures) features(w http.ResponseWriter, r *http.Request, collectionID string, - cursor domain.Cursor, limit int, fc *domain.FeatureCollection) { + cursor domain.Cursors, featuresURL featureCollectionURL, limit int, fc *domain.FeatureCollection) { collectionMetadata := collectionsMetadata[collectionID] @@ -59,11 +65,11 @@ func (hf *htmlFeatures) features(w http.ResponseWriter, r *http.Request, collect breadcrumbs = append(breadcrumbs, []engine.Breadcrumb{ { Name: getCollectionTitle(collectionID, collectionMetadata), - Path: "collections/" + collectionID, + Path: collectionsCrumb + collectionID, }, { Name: "Items", - Path: "collections/" + collectionID + "/items", + Path: collectionsCrumb + collectionID + "/items", }, }...) @@ -72,6 +78,8 @@ func (hf *htmlFeatures) features(w http.ResponseWriter, r *http.Request, collect collectionID, collectionMetadata, cursor, + featuresURL.toPrevNextURL(collectionID, cursor.Prev, engine.FormatHTML), + featuresURL.toPrevNextURL(collectionID, cursor.Next, engine.FormatHTML), limit, } @@ -86,15 +94,15 @@ func (hf *htmlFeatures) feature(w http.ResponseWriter, r *http.Request, collecti breadcrumbs = append(breadcrumbs, []engine.Breadcrumb{ { Name: getCollectionTitle(collectionID, collectionMetadata), - Path: "collections/" + collectionID, + Path: collectionsCrumb + collectionID, }, { Name: "Items", - Path: "collections/" + collectionID + "/items", + Path: collectionsCrumb + collectionID + "/items", }, { Name: strconv.FormatInt(feat.ID, 10), - Path: "collections/" + collectionID + "/items/" + strconv.FormatInt(feat.ID, 10), + Path: collectionsCrumb + collectionID + "/items/" + strconv.FormatInt(feat.ID, 10), }, }...) diff --git a/ogc/features/json.go b/ogc/features/json.go index 20d3c9a4..efd50094 100644 --- a/ogc/features/json.go +++ b/ogc/features/json.go @@ -3,7 +3,6 @@ package features import ( "bytes" "encoding/json" - "fmt" "net/http" "github.com/PDOK/gokoala/engine" @@ -21,9 +20,9 @@ func newJSONFeatures(e *engine.Engine) *jsonFeatures { } func (jf *jsonFeatures) featuresAsGeoJSON(w http.ResponseWriter, collectionID string, - cursor domain.Cursor, limit int, fc *domain.FeatureCollection) { + cursor domain.Cursors, featuresURL featureCollectionURL, fc *domain.FeatureCollection) { - fc.Links = jf.createFeatureCollectionLinks(collectionID, cursor, limit) + fc.Links = jf.createFeatureCollectionLinks(collectionID, cursor, featuresURL) fcJSON, err := toJSON(&fc) if err != nil { http.Error(w, "Failed to marshal FeatureCollection to JSON", http.StatusInternalServerError) @@ -32,8 +31,8 @@ func (jf *jsonFeatures) featuresAsGeoJSON(w http.ResponseWriter, collectionID st engine.SafeWrite(w.Write, fcJSON) } -func (jf *jsonFeatures) featureAsGeoJSON(w http.ResponseWriter, collectionID string, feat *domain.Feature) { - feat.Links = jf.createFeatureLinks(collectionID, feat.ID) +func (jf *jsonFeatures) featureAsGeoJSON(w http.ResponseWriter, collectionID string, feat *domain.Feature, url featureURL) { + feat.Links = jf.createFeatureLinks(url, collectionID, feat.ID) featJSON, err := toJSON(feat) if err != nil { http.Error(w, "Failed to marshal Feature to JSON", http.StatusInternalServerError) @@ -50,62 +49,58 @@ func (jf *jsonFeatures) featureAsJSONFG() { // TODO: not implemented yet } -func (jf *jsonFeatures) createFeatureCollectionLinks(collectionID string, cursor domain.Cursor, limit int) []domain.Link { - featuresBaseURL := fmt.Sprintf("%s/collections/%s/items", jf.engine.Config.BaseURL.String(), collectionID) - +func (jf *jsonFeatures) createFeatureCollectionLinks(collectionID string, cursor domain.Cursors, featuresURL featureCollectionURL) []domain.Link { links := make([]domain.Link, 0) links = append(links, domain.Link{ Rel: "self", Title: "This document as GeoJSON", Type: engine.MediaTypeGeoJSON, - Href: featuresBaseURL + "?f=json", + Href: featuresURL.toSelfURL(collectionID, engine.FormatJSON), }) links = append(links, domain.Link{ Rel: "alternate", Title: "This document as HTML", Type: engine.MediaTypeHTML, - Href: featuresBaseURL + "?f=html", + Href: featuresURL.toSelfURL(collectionID, engine.FormatHTML), }) - if !cursor.IsLast { + if cursor.HasNext { links = append(links, domain.Link{ Rel: "next", Title: "Next page", Type: engine.MediaTypeGeoJSON, - Href: fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Next, limit), + Href: featuresURL.toPrevNextURL(collectionID, cursor.Next, engine.FormatJSON), }) } - if !cursor.IsFirst { + if cursor.HasPrev { links = append(links, domain.Link{ Rel: "prev", Title: "Previous page", Type: engine.MediaTypeGeoJSON, - Href: fmt.Sprintf("%s?f=json&cursor=%s&limit=%d", featuresBaseURL, cursor.Prev, limit), + Href: featuresURL.toPrevNextURL(collectionID, cursor.Prev, engine.FormatJSON), }) } return links } -func (jf *jsonFeatures) createFeatureLinks(collectionID string, featureID int64) []domain.Link { - featureBaseURL := fmt.Sprintf("%s/collections/%s/items/%d", jf.engine.Config.BaseURL.String(), collectionID, featureID) - +func (jf *jsonFeatures) createFeatureLinks(url featureURL, collectionID string, featureID int64) []domain.Link { links := make([]domain.Link, 0) links = append(links, domain.Link{ Rel: "self", Title: "This document as GeoJSON", Type: engine.MediaTypeGeoJSON, - Href: featureBaseURL + "?f=json", + Href: url.toSelfURL(collectionID, featureID, engine.FormatJSON), }) links = append(links, domain.Link{ Rel: "alternate", Title: "This document as HTML", Type: engine.MediaTypeHTML, - Href: featureBaseURL + "?f=html", + Href: url.toSelfURL(collectionID, featureID, engine.FormatHTML), }) links = append(links, domain.Link{ Rel: "collection", Title: "The collection to which this feature belongs", Type: engine.MediaTypeJSON, - Href: fmt.Sprintf("%s/collections/%s?f=json", jf.engine.Config.BaseURL.String(), collectionID), + Href: url.toCollectionURL(collectionID, engine.FormatJSON), }) return links } diff --git a/ogc/features/main.go b/ogc/features/main.go index 346b946d..260a5811 100644 --- a/ogc/features/main.go +++ b/ogc/features/main.go @@ -10,8 +10,8 @@ import ( "github.com/PDOK/gokoala/engine" "github.com/PDOK/gokoala/ogc/common/geospatial" "github.com/PDOK/gokoala/ogc/features/datasources" - "github.com/PDOK/gokoala/ogc/features/datasources/fakedb" "github.com/PDOK/gokoala/ogc/features/datasources/geopackage" + "github.com/PDOK/gokoala/ogc/features/datasources/postgis" "github.com/PDOK/gokoala/ogc/features/domain" "github.com/go-chi/chi/v5" ) @@ -34,13 +34,13 @@ type Features struct { } func NewFeatures(e *engine.Engine, router *chi.Mux) *Features { + cfg := e.Config.OgcAPI.Features + var datasource datasources.Datasource - if e.Config.OgcAPI.Features.Datasource.FakeDB { - datasource = fakedb.NewFakeDB() - } else if e.Config.OgcAPI.Features.Datasource.GeoPackage != nil { - datasource = geopackage.NewGeoPackage( - e.Config.OgcAPI.Features.Collections, - *e.Config.OgcAPI.Features.Datasource.GeoPackage) + if cfg.Datasource.GeoPackage != nil { + datasource = geopackage.NewGeoPackage(cfg.Collections, *cfg.Datasource.GeoPackage) + } else if cfg.Datasource.PostGIS != nil { + datasource = postgis.NewPostGIS() } e.RegisterShutdownHook(datasource.Close) @@ -61,13 +61,14 @@ func NewFeatures(e *engine.Engine, router *chi.Mux) *Features { func (f *Features) CollectionContent() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { collectionID := chi.URLParam(r, "collectionId") - encodedCursor := domain.EncodedCursor(r.URL.Query().Get("cursor")) + encodedCursor := domain.EncodedCursor(r.URL.Query().Get(cursorParam)) limit, err := getLimit(r) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } - if err = f.validateNoUnknownFeatureCollectionQueryParams(r); err != nil { + url := featureCollectionURL{*f.engine.Config.BaseURL.URL, r.URL.Query()} + if err = url.validateNoUnknownParams(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -76,12 +77,11 @@ func (f *Features) CollectionContent() http.HandlerFunc { return } - params := datasources.QueryParams{ - Cursor: encodedCursor.Decode(), + fc, newCursor, err := f.datasource.GetFeatures(r.Context(), collectionID, datasources.FeatureOptions{ + Cursor: encodedCursor.Decode(url.checksum()), Limit: limit, // TODO set bbox, bbox-crs, etc - } - fc, cursor, err := f.datasource.GetFeatures(r.Context(), collectionID, params) + }) if err != nil { // log error, but sent generic message to client to prevent possible information leakage from datasource msg := fmt.Sprintf("failed to retrieve feature collection %s", collectionID) @@ -93,12 +93,11 @@ func (f *Features) CollectionContent() http.HandlerFunc { return } - format := f.engine.CN.NegotiateFormat(r) - switch format { + switch f.engine.CN.NegotiateFormat(r) { case engine.FormatHTML: - f.html.features(w, r, collectionID, cursor, limit, fc) + f.html.features(w, r, collectionID, newCursor, url, limit, fc) case engine.FormatJSON: - f.json.featuresAsGeoJSON(w, collectionID, cursor, limit, fc) + f.json.featuresAsGeoJSON(w, collectionID, newCursor, url, fc) case engine.FormatJSONFG: f.json.featuresAsJSONFG() default: @@ -116,7 +115,8 @@ func (f *Features) Feature() http.HandlerFunc { http.Error(w, "feature ID must be a number", http.StatusBadRequest) return } - if err = f.validateNoUnknownFeatureQueryParams(r); err != nil { + url := featureURL{*f.engine.Config.BaseURL.URL, r.URL.Query()} + if err = url.validateNoUnknownParams(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -134,12 +134,11 @@ func (f *Features) Feature() http.HandlerFunc { return } - format := f.engine.CN.NegotiateFormat(r) - switch format { + switch f.engine.CN.NegotiateFormat(r) { case engine.FormatHTML: f.html.feature(w, r, collectionID, feat) case engine.FormatJSON: - f.json.featureAsGeoJSON(w, collectionID, feat) + f.json.featureAsGeoJSON(w, collectionID, feat, url) case engine.FormatJSONFG: f.json.featureAsJSONFG() default: @@ -159,8 +158,8 @@ func (f *Features) cacheCollectionsMetadata() map[string]*engine.GeoSpatialColle func getLimit(r *http.Request) (int, error) { limit := defaultLimit var err error - if r.URL.Query().Get("limit") != "" { - limit, err = strconv.Atoi(r.URL.Query().Get("limit")) + if r.URL.Query().Get(limitParam) != "" { + limit, err = strconv.Atoi(r.URL.Query().Get(limitParam)) if err != nil { err = errors.New("limit query parameter must be a number") } @@ -170,32 +169,3 @@ func getLimit(r *http.Request) (int, error) { } return limit, err } - -// validateNoUnknownFeatureCollectionQueryParams implements req 7.6 (https://docs.ogc.org/is/17-069r4/17-069r4.html#query_parameters) -func (f *Features) validateNoUnknownFeatureCollectionQueryParams(r *http.Request) error { - copyQueryString := r.URL.Query() - copyQueryString.Del("f") - copyQueryString.Del("limit") - copyQueryString.Del("cursor") - copyQueryString.Del("datetime") - copyQueryString.Del("crs") - copyQueryString.Del("bbox") - copyQueryString.Del("bbox-crs") - copyQueryString.Del("filter") - copyQueryString.Del("filter-crs") - if len(copyQueryString) > 0 { - return fmt.Errorf("unknown query parameter(s) found: %v", copyQueryString.Encode()) - } - return nil -} - -// validateNoUnknownFeatureQueryParams implements req 7.6 (https://docs.ogc.org/is/17-069r4/17-069r4.html#query_parameters) -func (f *Features) validateNoUnknownFeatureQueryParams(r *http.Request) error { - copyQueryString := r.URL.Query() - copyQueryString.Del("f") - copyQueryString.Del("crs") - if len(copyQueryString) > 0 { - return fmt.Errorf("unknown query parameter(s) found: %v", copyQueryString.Encode()) - } - return nil -} diff --git a/ogc/features/main_test.go b/ogc/features/main_test.go index 7d9071d2..24ee3371 100644 --- a/ogc/features/main_test.go +++ b/ogc/features/main_test.go @@ -13,7 +13,6 @@ import ( "testing" "github.com/PDOK/gokoala/engine" - "github.com/brianvoe/gofakeit/v6" "github.com/go-chi/chi/v5" "github.com/stretchr/testify/assert" ) @@ -74,7 +73,7 @@ func TestFeatures_CollectionContent(t *testing.T) { name: "Request GeoJSON for 'foo' collection using limit of 2 and cursor to next page", fields: fields{ configFile: "ogc/features/testdata/config_features.yaml", - url: "http://localhost:8080/collections/tunneldelen/items?f=json&cursor=iUMnUmcz&limit=2", + url: "http://localhost:8080/collections/tunneldelen/items?f=json&cursor=Dv58Nwyr1Q%3D%3D&limit=2", collectionID: "foo", format: "json", }, @@ -151,8 +150,6 @@ func TestFeatures_CollectionContent(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gofakeit.Seed(1) // Uses consistent fake data. - req, err := createRequest(tt.fields.url, tt.fields.collectionID, "", tt.fields.format) if err != nil { log.Fatal(err) @@ -171,6 +168,8 @@ func TestFeatures_CollectionContent(t *testing.T) { if err != nil { log.Fatal(err) } + + log.Print(rr.Body.String()) // to ease debugging switch { case tt.fields.format == "json": assert.JSONEq(t, string(expectedBody), rr.Body.String()) @@ -202,16 +201,16 @@ func TestFeatures_Feature(t *testing.T) { want want }{ { - name: "Request GeoJSON for feature 19", + name: "Request GeoJSON for feature 4030", fields: fields{ configFile: "ogc/features/testdata/config_features.yaml", url: "http://localhost:8080/collections/:collectionId/items/:featureId", collectionID: "foo", - featureID: "19", + featureID: "4030", format: "json", }, want: want{ - body: "ogc/features/testdata/expected_feature_19.json", + body: "ogc/features/testdata/expected_feature_4030.json", statusCode: http.StatusOK, }, }, @@ -244,24 +243,22 @@ func TestFeatures_Feature(t *testing.T) { }, }, { - name: "Request HTML for feature 19", + name: "Request HTML for feature 4030", fields: fields{ configFile: "ogc/features/testdata/config_features.yaml", url: "http://localhost:8080/collections/:collectionId/items/:featureId", collectionID: "foo", - featureID: "19", + featureID: "4030", format: "html", }, want: want{ - body: "ogc/features/testdata/expected_feature_19.html", + body: "ogc/features/testdata/expected_feature_4030.html", statusCode: http.StatusOK, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gofakeit.Seed(1) // Uses consistent fake data. - req, err := createRequest(tt.fields.url, tt.fields.collectionID, tt.fields.featureID, tt.fields.format) if err != nil { log.Fatal(err) @@ -280,6 +277,8 @@ func TestFeatures_Feature(t *testing.T) { if err != nil { log.Fatal(err) } + + log.Print(rr.Body.String()) // to ease debugging switch { case tt.fields.format == "json": assert.JSONEq(t, string(expectedBody), rr.Body.String()) @@ -318,7 +317,7 @@ func createRequest(url string, collectionID string, featureID string, format str rctx.URLParams.Add("featureId", featureID) queryString := req.URL.Query() - queryString.Add("f", format) + queryString.Add(engine.FormatParam, format) req.URL.RawQuery = queryString.Encode() req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) diff --git a/ogc/features/templates/features.go.html b/ogc/features/templates/features.go.html index 4032b9b4..2db5f6f5 100644 --- a/ogc/features/templates/features.go.html +++ b/ogc/features/templates/features.go.html @@ -6,6 +6,7 @@ {{- /* generic function to update query string parameters */ -}} function updateQueryString(name, value) { const url = new URL(window.location.href); + url.searchParams.delete('cursor'); // when filters change, we can't continue pagination. url.searchParams.set(name, value); window.location.href = url.toString(); } @@ -41,13 +42,13 @@