Merge pull request #66 from PDOK/datasourceid

Allow user to configure "datasourceId" to map collections to tables

engine/config.go

@@ -209,7 +209,8 @@ type CollectionEntryTiles struct {
 }
 
 type CollectionEntryFeatures struct {
-	// placeholder
+	// Optional way to map a collection ID to the underlying datasource (e.g. table in database).
+	DatasourceID *string `yaml:"datasourceId"`
 }
 
 type CollectionEntryMaps struct {

engine/engine.go

@@ -38,9 +38,7 @@ type Engine struct {
 
 // NewEngine builds a new Engine
 func NewEngine(configFile string, openAPIFile string) *Engine {
-	config := readConfigFile(configFile)
-
-	return NewEngineWithConfig(config, openAPIFile)
+	return NewEngineWithConfig(readConfigFile(configFile), openAPIFile)
 }
 
 // NewEngineWithConfig builds a new Engine
@@ -87,7 +85,6 @@ func (e *Engine) startServer(name string, address string, shutdownDelay int, rou
 
 		ReadTimeout:       15 * time.Second,
 		ReadHeaderTimeout: 15 * time.Second,
-		WriteTimeout:      30 * time.Second,
 	}
 
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)

examples/config_features_azure.yaml

@@ -49,9 +49,10 @@ ogcApi:
           file: addresses.gpkg
           fid: fid
     collections:
-      - id: addresses
+      - id: dutch-addresses
+        datasourceId: addresses  # name of the feature table (optional), when omitted collection ID is used.
         metadata:
-          title: Adressen
+          title: Dutch Addresses
           description: These are example addresses
           keywords:
             - Building

examples/config_features_local.yaml

@@ -45,9 +45,10 @@ ogcApi:
           fid: fid
           queryTimeout: 5s
     collections:
-      - id: addresses
+      - id: dutch-addresses
+        datasourceId: addresses  # name of the feature table (optional), when omitted collection ID is used.
         metadata:
-          title: Adressen
+          title: Dutch Addresses
           description: These are example addresses
           keywords:
             - Building

examples/docker-compose-features-azure.yaml

@@ -52,4 +52,4 @@ services:
     image: alpine/curl
     depends_on:
       - gokoala
-    command: "-fsSL http://gokoala:8080/collections/addresses/items?f=json&limit=2"
+    command: "-fsSL http://gokoala:8080/collections/dutch-addresses/items?f=json&limit=2"

ogc/features/datasources/geopackage/geopackage.go

@@ -51,12 +51,12 @@ type featureTable struct {
 type GeoPackage struct {
 	backend geoPackageBackend
 
-	fidColumn        string
-	featureTableByID map[string]*featureTable
-	queryTimeout     time.Duration
+	fidColumn                  string
+	featureTableByCollectionID map[string]*featureTable
+	queryTimeout               time.Duration
 }
 
-func NewGeoPackage(gpkgConfig engine.GeoPackage) *GeoPackage {
+func NewGeoPackage(collections engine.GeoSpatialCollections, gpkgConfig engine.GeoPackage) *GeoPackage {
 	g := &GeoPackage{}
 	switch {
 	case gpkgConfig.Local != nil:
@@ -71,11 +71,11 @@ func NewGeoPackage(gpkgConfig engine.GeoPackage) *GeoPackage {
 		log.Fatal("unknown geopackage config encountered")
 	}
 
-	featureTables, err := readGpkgContents(g.backend.getDB())
+	featureTables, err := readGpkgContents(collections, g.backend.getDB())
 	if err != nil {
 		log.Fatal(err)
 	}
-	g.featureTableByID = featureTables
+	g.featureTableByCollectionID = featureTables
 
 	return g
 }
@@ -85,16 +85,16 @@ func (g *GeoPackage) Close() {
 }
 
 func (g *GeoPackage) GetFeatures(ctx context.Context, collection string, params datasources.QueryParams) (*domain.FeatureCollection, domain.Cursor, error) {
-	featureTable, ok := g.featureTableByID[collection]
+	table, ok := g.featureTableByCollectionID[collection]
 	if !ok {
 		return nil, domain.Cursor{}, fmt.Errorf("can't query collection '%s' since it doesn't exist in "+
-			"geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByID))
+			"geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByCollectionID))
 	}
 
 	queryCtx, cancel := context.WithTimeout(ctx, g.queryTimeout) // https://go.dev/doc/database/cancel-operations
 	defer cancel()
 
-	query, queryArgs := g.makeFeaturesQuery(featureTable, params)
+	query, queryArgs := g.makeFeaturesQuery(table, params)
 	stmt, err := g.backend.getDB().PreparexContext(ctx, query)
 	if err != nil {
 		return nil, domain.Cursor{}, err
@@ -108,7 +108,7 @@ func (g *GeoPackage) GetFeatures(ctx context.Context, collection string, params
 	defer rows.Close()
 
 	result := domain.FeatureCollection{}
-	result.Features, err = domain.MapRowsToFeatures(rows, g.fidColumn, featureTable.GeometryColumnName, readGpkgGeometry)
+	result.Features, err = domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry)
 	if err != nil {
 		return nil, domain.Cursor{}, err
 	}
@@ -120,16 +120,16 @@ func (g *GeoPackage) GetFeatures(ctx context.Context, collection string, params
 }
 
 func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureID int64) (*domain.Feature, error) {
-	gpkgContent, ok := g.featureTableByID[collection]
+	table, ok := g.featureTableByCollectionID[collection]
 	if !ok {
 		return nil, fmt.Errorf("can't query collection '%s' since it doesn't exist in "+
-			"geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByID))
+			"geopackage, available in geopackage: %v", collection, engine.Keys(g.featureTableByCollectionID))
 	}
 
 	queryCtx, cancel := context.WithTimeout(ctx, g.queryTimeout) // https://go.dev/doc/database/cancel-operations
 	defer cancel()
 
-	query := fmt.Sprintf("select * from %s f where f.%s = ? limit 1", gpkgContent.TableName, g.fidColumn)
+	query := fmt.Sprintf("select * from %s f where f.%s = ? limit 1", table.TableName, g.fidColumn)
 	stmt, err := g.backend.getDB().PreparexContext(ctx, query)
 	if err != nil {
 		return nil, err
@@ -142,7 +142,7 @@ func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureI
 	}
 	defer rows.Close()
 
-	features, err := domain.MapRowsToFeatures(rows, g.fidColumn, gpkgContent.GeometryColumnName, readGpkgGeometry)
+	features, err := domain.MapRowsToFeatures(rows, g.fidColumn, table.GeometryColumnName, readGpkgGeometry)
 	if err != nil {
 		return nil, err
 	}
@@ -152,17 +152,21 @@ func (g *GeoPackage) GetFeature(ctx context.Context, collection string, featureI
 	return features[0], nil
 }
 
-func (g *GeoPackage) makeFeaturesQuery(ft *featureTable, params datasources.QueryParams) (string, []any) {
+func (g *GeoPackage) makeFeaturesQuery(table *featureTable, params datasources.QueryParams) (string, []any) {
 	if params.Bbox != nil {
 		// TODO create bbox query
 		bboxQuery := ""
 		return bboxQuery, []any{params.Cursor, params.Limit, params.Bbox}
 	}
-	defaultQuery := fmt.Sprintf("select * from %s f where f.%s > ? order by f.%s limit ?", ft.TableName, g.fidColumn, g.fidColumn)
+	defaultQuery := fmt.Sprintf("select * from %s f where f.%s > ? order by f.%s limit ?", table.TableName, g.fidColumn, g.fidColumn)
 	return defaultQuery, []any{params.Cursor, params.Limit}
 }
 
-func readGpkgContents(db *sqlx.DB) (map[string]*featureTable, error) {
+// Read gpkg_contents table. This table contains metadata about feature tables. The result is a mapping from
+// collection ID -> feature table metadata. We match each feature table to the collection ID by looking at the
+// 'identifier' column. Also in case there's no exact match between 'collection ID' and 'identifier' we use
+// the explicitly configured 'datasource ID'
+func readGpkgContents(collections engine.GeoSpatialCollections, db *sqlx.DB) (map[string]*featureTable, error) {
 	rows, err := db.Queryx(queryGpkgContent)
 	if err != nil {
 		return nil, fmt.Errorf("failed to retrieve gpkg_contents using query: %v\n, error: %w", queryGpkgContent, err)
@@ -178,7 +182,20 @@ func readGpkgContents(db *sqlx.DB) (map[string]*featureTable, error) {
 		if row.TableName == "" {
 			return nil, fmt.Errorf("feature table name is blank, error: %w", err)
 		}
-		result[row.Identifier] = &row
+
+		if len(collections) == 0 {
+			result[row.Identifier] = &row
+		} else {
+			for _, collection := range collections {
+				if row.Identifier == collection.ID {
+					result[collection.ID] = &row
+					break
+				} else if collection.Features.DatasourceID != nil && row.Identifier == *collection.Features.DatasourceID {
+					result[collection.ID] = &row
+					break
+				}
+			}
+		}
 	}
 
 	if err := rows.Err(); err != nil {

ogc/features/datasources/geopackage/geopackage_test.go

@@ -53,7 +53,7 @@ func TestNewGeoPackage(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equalf(t, tt.wantNrOfFeatureTablesInGpkg, len(NewGeoPackage(tt.args.config).featureTableByID), "NewGeoPackage(%v)", tt.args.config)
+			assert.Equalf(t, tt.wantNrOfFeatureTablesInGpkg, len(NewGeoPackage(nil, tt.args.config).featureTableByCollectionID), "NewGeoPackage(%v)", tt.args.config)
 		})
 	}
 }
@@ -196,10 +196,10 @@ func TestGeoPackage_GetFeatures(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			g := &GeoPackage{
-				backend:          tt.fields.backend,
-				fidColumn:        tt.fields.fidColumn,
-				featureTableByID: tt.fields.featureTableByID,
-				queryTimeout:     tt.fields.queryTimeout,
+				backend:                    tt.fields.backend,
+				fidColumn:                  tt.fields.fidColumn,
+				featureTableByCollectionID: tt.fields.featureTableByID,
+				queryTimeout:               tt.fields.queryTimeout,
 			}
 			fc, cursor, err := g.GetFeatures(tt.args.ctx, tt.args.collection, tt.args.queryParams)
 			if err != nil {
@@ -300,10 +300,10 @@ func TestGeoPackage_GetFeature(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			g := &GeoPackage{
-				backend:          tt.fields.backend,
-				fidColumn:        tt.fields.fidColumn,
-				featureTableByID: tt.fields.featureTableByID,
-				queryTimeout:     tt.fields.queryTimeout,
+				backend:                    tt.fields.backend,
+				fidColumn:                  tt.fields.fidColumn,
+				featureTableByCollectionID: tt.fields.featureTableByID,
+				queryTimeout:               tt.fields.queryTimeout,
 			}
 			got, err := g.GetFeature(tt.args.ctx, tt.args.collection, tt.args.featureID)
 			if err != nil {

ogc/features/main.go

@@ -38,7 +38,9 @@ func NewFeatures(e *engine.Engine, router *chi.Mux) *Features {
 	if e.Config.OgcAPI.Features.Datasource.FakeDB {
 		datasource = fakedb.NewFakeDB()
 	} else if e.Config.OgcAPI.Features.Datasource.GeoPackage != nil {
-		datasource = geopackage.NewGeoPackage(*e.Config.OgcAPI.Features.Datasource.GeoPackage)
+		datasource = geopackage.NewGeoPackage(
+			e.Config.OgcAPI.Features.Collections,
+			*e.Config.OgcAPI.Features.Datasource.GeoPackage)
 	}
 	e.RegisterShutdownHook(datasource.Close)