Merge pull request #161 from PDOK/PDOK/geovolumes-proxy-update

Validate 3dgeovolumes responses (optionally)

config/config.go

@@ -463,6 +463,13 @@ type OgcAPI3dGeoVolumes struct {
 
 	// Collections to be served as 3D GeoVolumes
 	Collections GeoSpatialCollections `yaml:"collections" json:"collections"`
+
+	// Whether JSON responses will be validated against the OpenAPI spec
+	// since it has significant performance impact when dealing with large JSON payloads.
+	//
+	// +kubebuilder:default=true
+	// +optional
+	ValidateResponses *bool `yaml:"validateResponses,omitempty" json:"validateResponses,omitempty" default:"true"` // ptr due to https://github.com/creasty/defaults/issues/49
 }
 
 // +kubebuilder:validation:Enum=raster;vector

engine/contentnegotiation.go

@@ -34,6 +34,7 @@ const (
 	FormatSLD            = "sld10"
 	FormatGeoJSON        = "geojson" // ?=json should also work for geojson
 	FormatJSONFG         = "jsonfg"
+	FormatGzip           = "gzip"
 )
 
 var (

engine/engine.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -27,15 +28,16 @@ const (
 	templatesDir    = "engine/templates/"
 	shutdownTimeout = 5 * time.Second
 
-	HeaderLink           = "Link"
-	HeaderAccept         = "Accept"
-	HeaderAcceptLanguage = "Accept-Language"
-	HeaderContentType    = "Content-Type"
-	HeaderContentLength  = "Content-Length"
-	HeaderContentCrs     = "Content-Crs"
-	HeaderBaseURL        = "X-BaseUrl"
-	HeaderRequestedWith  = "X-Requested-With"
-	HeaderAPIVersion     = "API-Version"
+	HeaderLink            = "Link"
+	HeaderAccept          = "Accept"
+	HeaderAcceptLanguage  = "Accept-Language"
+	HeaderContentType     = "Content-Type"
+	HeaderContentLength   = "Content-Length"
+	HeaderContentCrs      = "Content-Crs"
+	HeaderContentEncoding = "Content-Encoding"
+	HeaderBaseURL         = "X-BaseUrl"
+	HeaderRequestedWith   = "X-Requested-With"
+	HeaderAPIVersion      = "API-Version"
 )
 
 // Engine encapsulates shared non-OGC API specific logic
@@ -299,6 +301,12 @@ func (e *Engine) ServeResponse(w http.ResponseWriter, r *http.Request,
 // ReverseProxy forwards given HTTP request to given target server, and optionally tweaks response
 func (e *Engine) ReverseProxy(w http.ResponseWriter, r *http.Request, target *url.URL,
 	prefer204 bool, contentTypeOverwrite string) {
+	e.ReverseProxyAndValidate(w, r, target, prefer204, contentTypeOverwrite, false)
+}
+
+// ReverseProxy forwards given HTTP request to given target server, and optionally tweaks and validates response
+func (e *Engine) ReverseProxyAndValidate(w http.ResponseWriter, r *http.Request, target *url.URL,
+	prefer204 bool, contentTypeOverwrite string, validateResponse bool) {
 
 	rewrite := func(r *httputil.ProxyRequest) {
 		r.Out.URL = target
@@ -320,6 +328,26 @@ func (e *Engine) ReverseProxy(w http.ResponseWriter, r *http.Request, target *ur
 		if contentTypeOverwrite != "" {
 			proxyRes.Header.Set(HeaderContentType, contentTypeOverwrite)
 		}
+		if contentType := proxyRes.Header.Get(HeaderContentType); contentType == MediaTypeJSON && validateResponse {
+			var reader io.ReadCloser
+			var err error
+			if proxyRes.Header.Get(HeaderContentEncoding) == FormatGzip {
+				reader, err = gzip.NewReader(proxyRes.Body)
+				if err != nil {
+					log.Printf("%v", err.Error())
+					return err
+				}
+			} else {
+				reader = proxyRes.Body
+			}
+			defer reader.Close()
+			res, err := io.ReadAll(reader)
+			if err != nil {
+				log.Printf("%v", err.Error())
+				return err
+			}
+			e.ServeResponse(w, r, false, true, contentType, res)
+		}
 		return nil
 	}
 

engine/engine_test.go

@@ -73,6 +73,28 @@ func TestEngine_ReverseProxy(t *testing.T) {
 	assert.Equal(t, rec.Body.String(), "Mock response, received header https://api.foobar.example/")
 }
 
+func TestEngine_ReverseProxyAndValidate(t *testing.T) {
+	// given
+	mockTargetServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, err := w.Write([]byte("Mock response, received header " + r.Header.Get(HeaderBaseURL)))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}))
+	defer mockTargetServer.Close()
+
+	engine, targetURL := makeEngine(mockTargetServer)
+	rec, req := makeAPICall(t, mockTargetServer)
+
+	// when
+	engine.ReverseProxyAndValidate(rec, req, targetURL, false, MediaTypeJSON, true)
+
+	// then
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "Mock response, received header https://api.foobar.example/", rec.Body.String())
+}
+
 func TestEngine_ReverseProxy_Status204(t *testing.T) {
 	// given
 	mockTargetServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
@@ -92,10 +114,13 @@ func TestEngine_ReverseProxy_Status204(t *testing.T) {
 }
 
 func makeEngine(mockTargetServer *httptest.Server) (*Engine, *url.URL) {
+	cfg := &config.Config{
+		BaseURL: config.URL{URL: &url.URL{Scheme: "https", Host: "api.foobar.example", Path: "/"}},
+	}
+	openAPI := newOpenAPI(cfg, []string{""}, nil)
 	engine := &Engine{
-		Config: &config.Config{
-			BaseURL: config.URL{URL: &url.URL{Scheme: "https", Host: "api.foobar.example", Path: "/"}},
-		},
+		Config:  cfg,
+		OpenAPI: openAPI,
 	}
 	targetURL, _ := url.Parse(mockTargetServer.URL)
 	return engine, targetURL

engine/templates/openapi/3dgeovolumes.go.json

@@ -245,7 +245,7 @@
         }
       },
       "Tile_Implicit" : {
-        "required" : [ "boundingVolume", "content", "implicitTiling", "refine" ],
+        "required" : [ "boundingVolume", "geometricError", "implicitTiling", "refine" ],
         "type" : "object",
         "properties" : {
           "content" : {
@@ -262,11 +262,43 @@
           },
           "implicitTiling" : {
             "$ref" : "#/components/schemas/ImplicitTiling"
+          },
+          "children" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/Tile_Children"
+            },
+            "minItems": 1
           }
         }
       },
       "Tile_Explicit" : {
-        "required" : [ "boundingVolume", "content", "refine" ],
+        "required" : [ "boundingVolume", "geometricError", "refine" ],
+        "type" : "object",
+        "properties" : {
+          "content" : {
+            "$ref" : "#/components/schemas/Content"
+          },
+          "boundingVolume" : {
+            "$ref" : "#/components/schemas/BoundingVolume"
+          },
+          "geometricError" : {
+            "type" : "number"
+          },
+          "refine" : {
+            "type" : "string"
+          },
+          "children" : {
+            "type" : "array",
+            "items" : {
+              "$ref" : "#/components/schemas/Tile_Children"
+            },
+            "minItems": 1
+          }
+        }
+      },
+      "Tile_Children" : {
+        "required" : [ "boundingVolume", "geometricError" ],
         "type" : "object",
         "properties" : {
           "content" : {
@@ -284,7 +316,7 @@
           "children" : {
             "type" : "array",
             "items" : {
-              "$ref" : "#/components/schemas/Tile_Explicit"
+              "$ref" : "#/components/schemas/Tile_Children"
             },
             "minItems": 1
           }

ogc/geovolumes/main.go

@@ -16,7 +16,8 @@ import (
 )
 
 type ThreeDimensionalGeoVolumes struct {
-	engine *engine.Engine
+	engine           *engine.Engine
+	validateResponse bool
 }
 
 func NewThreeDimensionalGeoVolumes(e *engine.Engine) *ThreeDimensionalGeoVolumes {
@@ -26,7 +27,8 @@ func NewThreeDimensionalGeoVolumes(e *engine.Engine) *ThreeDimensionalGeoVolumes
 	}
 
 	geoVolumes := &ThreeDimensionalGeoVolumes{
-		engine: e,
+		engine:           e,
+		validateResponse: *e.Config.OgcAPI.GeoVolumes.ValidateResponses,
 	}
 
 	// 3D Tiles
@@ -130,7 +132,7 @@ func (t *ThreeDimensionalGeoVolumes) reverseProxy(w http.ResponseWriter, r *http
 		engine.RenderProblem(engine.ProblemServerError, w)
 		return
 	}
-	t.engine.ReverseProxy(w, r, target, prefer204, contentTypeOverwrite)
+	t.engine.ReverseProxyAndValidate(w, r, target, prefer204, contentTypeOverwrite, t.validateResponse)
 }
 
 func (t *ThreeDimensionalGeoVolumes) idToCollection(cid string) (*config.GeoSpatialCollection, error) {