feat: Add sitemap + structured data for SEO

.golangci.yaml

@@ -82,6 +82,7 @@ linters:
     - gosec  # inspects source code for security problems
     - loggercheck  # checks key value pairs for common logger libraries (kitlog,klog,logr,zap)
     - makezero  # finds slice declarations with non-zero initial length
+    - misspell  # finds commonly misspelled English words
     - nakedret  # finds naked returns in functions greater than a specified function length
     - nestif  # reports deeply nested if statements
     - nilerr  # finds the code that returns nil even if it checks that the error is not nil

README.md

@@ -173,6 +173,7 @@ already being taken care of when building the Docker container image.
   - `"*.go.html"`
   - `"*.go.json"`
   - `"*.go.tilejson"`
+  - `"*.go.xml"`
 - Now add template language support by running the
   [setup-jetbrains-gotemplates.sh](hack/setup-jetbrains-gotemplates.sh) script.
 - Reopen the project (or restart IDE). Now you'll have full IDE support in the GoKoala templates.
@@ -190,7 +191,8 @@ Also:
   - `"*.go.html"`
   - `"*.go.json"`
   - `"*.go.tilejson"`
-- Also add `html` and `json` to the list of Go template languages.
+  - `"*.go.xml"`
+- Also add `html`, `json` and `xml` to the list of Go template languages.
 - Now you'll have IDE support in the GoKoala templates.
 
 ### OGC compliance validation

cmd/main_test.go

@@ -80,6 +80,24 @@ func Test_newRouter(t *testing.T) {
 			apiCall:    "http://localhost:8181/conformance?f=html",
 			wantBody:   "internal/engine/testdata/expected_processes_conformance.html",
 		},
+		{
+			name:       "Should have valid sitemap XML",
+			configFile: "examples/config_all.yaml",
+			apiCall:    "http://localhost:8181/sitemap.xml",
+			wantBody:   "internal/engine/testdata/expected_sitemap.xml",
+		},
+		{
+			name:       "Should have valid structured data of type 'Dataset' on landing page",
+			configFile: "examples/config_all.yaml",
+			apiCall:    "http://localhost:8181?f=html",
+			wantBody:   "internal/engine/testdata/expected_dataset_landingpage.json",
+		},
+		{
+			name:       "Should have valid structured data of type 'Dataset' on (each) collection page",
+			configFile: "examples/config_all.yaml",
+			apiCall:    "http://localhost:8181/collections/addresses?f=html",
+			wantBody:   "internal/engine/testdata/expected_dataset_collection.json",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -107,7 +125,7 @@ func Test_newRouter(t *testing.T) {
 			switch {
 			case strings.HasSuffix(tt.apiCall, "json"):
 				assert.JSONEq(t, recorder.Body.String(), string(expectedBody))
-			case strings.HasSuffix(tt.apiCall, "html"):
+			case strings.HasSuffix(tt.apiCall, "html") || strings.HasSuffix(tt.apiCall, "xml"):
 				assert.Contains(t, normalize(recorder.Body.String()), normalize(string(expectedBody)))
 			default:
 				log.Fatalf("implement support to test format: %s", tt.apiCall)

config/collections.go

@@ -240,7 +240,7 @@ func sortByLiteralOrder(collections []GeoSpatialCollection, literalOrder []strin
 		collectionOrderIndex[id] = i
 	}
 	sort.Slice(collections, func(i, j int) bool {
-		// sort according to the explict/literal order specified in OgcAPICollectionOrder
+		// sort according to the explicit/literal order specified in OgcAPICollectionOrder
 		return collectionOrderIndex[collections[i].ID] < collectionOrderIndex[collections[j].ID]
 	})
 }

hack/setup-jetbrains-gotemplates.sh

@@ -23,6 +23,11 @@ traverse_templates() {
     echo "Processing file: $file"
     echo "<file url=\"file://\$PROJECT_DIR\$/$file\" dialect=\"JSON\" />" >> ".idea/templateLanguages.xml"
   done
+  # XML templates
+  find * -type f -iname "*.go.xml" -print0 | while IFS= read -r -d '' file; do
+    echo "Processing file: $file"
+    echo "<file url=\"file://\$PROJECT_DIR\$/$file\" dialect=\"XML\" />" >> ".idea/templateLanguages.xml"
+  done
 }
 
 mkdir -p ".idea/"

internal/engine/contentnegotiation.go

@@ -15,6 +15,7 @@ const (
 	languageParam = "lang"
 
 	MediaTypeJSON          = "application/json"
+	MediaTypeXML           = "application/xml"
 	MediaTypeHTML          = "text/html"
 	MediaTypeTileJSON      = "application/vnd.mapbox.tile+json"
 	MediaTypeMVT           = "application/vnd.mapbox-vector-tile"
@@ -26,6 +27,7 @@ const (
 	MediaTypeQuantizedMesh = "application/vnd.quantized-mesh"
 
 	FormatHTML           = "html"
+	FormatXML            = "xml"
 	FormatJSON           = "json"
 	FormatTileJSON       = "tilejson"
 	FormatMVT            = "mvt"
@@ -74,6 +76,7 @@ func newContentNegotiation(availableLanguages []config.Language) *ContentNegotia
 	availableMediaTypes := []contenttype.MediaType{
 		// in order
 		contenttype.NewMediaType(MediaTypeJSON),
+		contenttype.NewMediaType(MediaTypeXML),
 		contenttype.NewMediaType(MediaTypeHTML),
 		contenttype.NewMediaType(MediaTypeTileJSON),
 		contenttype.NewMediaType(MediaTypeGeoJSON),
@@ -85,6 +88,7 @@ func newContentNegotiation(availableLanguages []config.Language) *ContentNegotia
 
 	formatsByMediaType := map[string]string{
 		MediaTypeJSON:        FormatJSON,
+		MediaTypeXML:         FormatXML,
 		MediaTypeHTML:        FormatHTML,
 		MediaTypeTileJSON:    FormatTileJSON,
 		MediaTypeGeoJSON:     FormatGeoJSON,

internal/engine/contentnegotiation_test.go

@@ -23,7 +23,8 @@ func TestContentNegotiation_NegotiateFormat(t *testing.T) {
 	testFormat(t, cn, "application/json", "http://pdok.example/ogc/api.json", "json")
 	testFormat(t, cn, "application/json", "http://pdok.example/ogc/api?f=json", "json")
 	testFormat(t, cn, "", "http://pdok.example/ogc/api?f=json", "json")
-	testFormat(t, cn, "application/xml, application/json, text/css, text/html", "http://pdok.example/ogc/api/", "json")
+	testFormat(t, cn, "application/xml, application/json, text/css, text/html", "http://pdok.example/ogc/api/", "xml")
+	testFormat(t, cn, "application/json, application/xml, text/css, text/html", "http://pdok.example/ogc/api/", "json")
 	testLanguage(t, cn, "nl;q=1", "http://pdok.example/ogc/api", language.Dutch)
 	testLanguage(t, cn, "fr;q=0.8, de;q=0.5", "http://pdok.example/ogc/api", language.Dutch)
 	testLanguage(t, cn, "en;q=1", "http://pdok.example/ogc/api", language.English)

internal/engine/engine.go

@@ -77,12 +77,10 @@ func NewEngineWithConfig(config *config.Config, openAPIFile string, enableTraili
 		Router:    router,
 	}
 
-	if config.Resources != nil {
-		newResourcesEndpoint(engine) // Resources endpoint to serve static assets
-	}
-	router.Get("/health", func(w http.ResponseWriter, _ *http.Request) {
-		SafeWrite(w.Write, []byte("OK")) // Health endpoint
-	})
+	// Default (non-OGC) endpoints
+	newSitemap(engine)
+	newHealthEndpoint(engine)
+	newResourcesEndpoint(engine)
 	return engine
 }
 
@@ -172,42 +170,37 @@ func (e *Engine) ParseTemplate(key TemplateKey) {
 // This method also performs OpenAPI validation of the rendered template, therefore we also need the URL path.
 // The rendered templates are stored in the engine for future serving using ServePage.
 func (e *Engine) RenderTemplates(urlPath string, breadcrumbs []Breadcrumb, keys ...TemplateKey) {
-	for _, key := range keys {
-		e.Templates.renderAndSaveTemplate(key, breadcrumbs, nil)
-
-		// we already perform OpenAPI validation here during startup to catch
-		// issues early on, in addition to runtime OpenAPI response validation
-		// all templates are created in all available languages, hence all are checked
-		for lang := range e.Templates.localizers {
-			key.Language = lang
-			if err := e.validateStaticResponse(key, urlPath); err != nil {
-				log.Fatal(err)
-			}
-		}
-	}
+	e.renderTemplates(urlPath, nil, breadcrumbs, true, keys...)
 }
 
 // RenderTemplatesWithParams renders both HTMl and non-HTML templates depending on the format given in the TemplateKey.
 func (e *Engine) RenderTemplatesWithParams(urlPath string, params any, breadcrumbs []Breadcrumb, keys ...TemplateKey) {
+	e.renderTemplates(urlPath, params, breadcrumbs, true, keys...)
+}
+
+func (e *Engine) renderTemplates(urlPath string, params any, breadcrumbs []Breadcrumb, validate bool, keys ...TemplateKey) {
 	for _, key := range keys {
 		e.Templates.renderAndSaveTemplate(key, breadcrumbs, params)
 
-		// we already perform OpenAPI validation here during startup to catch
-		// issues early on, in addition to runtime OpenAPI response validation
-		// all templates are created in all available languages, hence all are checked
-		for lang := range e.Templates.localizers {
-			key.Language = lang
-			if err := e.validateStaticResponse(key, urlPath); err != nil {
-				log.Fatal(err)
+		if validate {
+			// we already perform OpenAPI validation here during startup to catch
+			// issues early on, in addition to runtime OpenAPI response validation
+			// all templates are created in all available languages, hence all are checked
+			for lang := range e.Templates.localizers {
+				key.Language = lang
+				if err := e.validateStaticResponse(key, urlPath); err != nil {
+					log.Fatal(err)
+				}
 			}
 		}
 	}
 }
 
-// RenderAndServePage renders an already parsed HTML or non-HTML template and renders it on-the-fly depending
+// RenderAndServePage renders an already parsed HTML or non-HTML template on-the-fly depending
 // on the format in the given TemplateKey. The result isn't store in engine, it's served directly to the client.
 //
-// NOTE: only used this for dynamic pages that can't be pre-rendered and cached (e.g. with data from a backing store).
+// NOTE: only used this for dynamic pages that can't be pre-rendered and cached (e.g. with data from a datastore),
+// otherwise use ServePage for pre-rendered pages.
 func (e *Engine) RenderAndServePage(w http.ResponseWriter, r *http.Request, key TemplateKey,
 	params any, breadcrumbs []Breadcrumb) {
 
@@ -242,49 +235,22 @@ func (e *Engine) RenderAndServePage(w http.ResponseWriter, r *http.Request, key
 		RenderProblem(ProblemServerError, w, err.Error())
 		return
 	}
-
-	// return response output to client
-	if contentType != "" {
-		w.Header().Set(HeaderContentType, contentType)
-	}
-	SafeWrite(w.Write, output)
+	writeResponse(w, contentType, output)
 }
 
 // ServePage serves a pre-rendered template while also validating against the OpenAPI spec
 func (e *Engine) ServePage(w http.ResponseWriter, r *http.Request, templateKey TemplateKey) {
-	// validate request
-	if err := e.OpenAPI.ValidateRequest(r); err != nil {
-		log.Printf("%v", err.Error())
-		RenderProblem(ProblemBadRequest, w, err.Error())
-		return
-	}
-
-	// render output
-	output, err := e.Templates.getRenderedTemplate(templateKey)
-	if err != nil {
-		log.Printf("%v", err.Error())
-		RenderProblem(ProblemNotFound, w)
-		return
-	}
-	contentType := e.CN.formatToMediaType(templateKey.Format)
-
-	// validate response
-	if err := e.OpenAPI.ValidateResponse(contentType, output, r); err != nil {
-		log.Printf("%v", err.Error())
-		RenderProblem(ProblemServerError, w, err.Error())
-		return
-	}
+	e.serve(w, r, &templateKey, true, true, "", nil)
+}
 
-	// return response output to client
-	if contentType != "" {
-		w.Header().Set(HeaderContentType, contentType)
-	}
-	SafeWrite(w.Write, output)
+// Serve serves the given response (arbitrary bytes) while also validating against the OpenAPI spec
+func (e *Engine) Serve(w http.ResponseWriter, r *http.Request,
+	validateRequest bool, validateResponse bool, contentType string, output []byte) {
+	e.serve(w, r, nil, validateRequest, validateResponse, contentType, output)
 }
 
-// ServeResponse serves the given response (arbitrary bytes) while also validating against the OpenAPI spec
-func (e *Engine) ServeResponse(w http.ResponseWriter, r *http.Request,
-	validateRequest bool, validateResponse bool, contentType string, response []byte) {
+func (e *Engine) serve(w http.ResponseWriter, r *http.Request, templateKey *TemplateKey,
+	validateRequest bool, validateResponse bool, contentType string, output []byte) {
 
 	if validateRequest {
 		if err := e.OpenAPI.ValidateRequest(r); err != nil {
@@ -294,19 +260,26 @@ func (e *Engine) ServeResponse(w http.ResponseWriter, r *http.Request,
 		}
 	}
 
-	if validateResponse {
-		if err := e.OpenAPI.ValidateResponse(contentType, response, r); err != nil {
+	if templateKey != nil {
+		// render output
+		var err error
+		output, err = e.Templates.getRenderedTemplate(*templateKey)
+		if err != nil {
 			log.Printf("%v", err.Error())
-			RenderProblem(ProblemServerError, w, err.Error())
+			RenderProblem(ProblemNotFound, w)
 			return
 		}
+		contentType = e.CN.formatToMediaType(templateKey.Format)
 	}
 
-	// return response output to client
-	if contentType != "" {
-		w.Header().Set(HeaderContentType, contentType)
+	if validateResponse {
+		if err := e.OpenAPI.ValidateResponse(contentType, output, r); err != nil {
+			log.Printf("%v", err.Error())
+			RenderProblem(ProblemServerError, w, err.Error())
+			return
+		}
 	}
-	SafeWrite(w.Write, response)
+	writeResponse(w, contentType, output)
 }
 
 // ReverseProxy forwards given HTTP request to given target server, and optionally tweaks response
@@ -359,7 +332,7 @@ func (e *Engine) ReverseProxyAndValidate(w http.ResponseWriter, r *http.Request,
 			if err != nil {
 				return err
 			}
-			e.ServeResponse(w, r, false, true, contentType, res)
+			e.Serve(w, r, false, true, contentType, res)
 		}
 		return nil
 	}
@@ -394,6 +367,14 @@ func (e *Engine) validateStaticResponse(key TemplateKey, urlPath string) error {
 	return nil
 }
 
+// return response output to client
+func writeResponse(w http.ResponseWriter, contentType string, output []byte) {
+	if contentType != "" {
+		w.Header().Set(HeaderContentType, contentType)
+	}
+	SafeWrite(w.Write, output)
+}
+
 // SafeWrite executes the given http.ResponseWriter.Write while logging errors
 func SafeWrite(write func([]byte) (int, error), body []byte) {
 	_, err := write(body)

internal/engine/health.go

@@ -0,0 +1,11 @@
+package engine
+
+import (
+	"net/http"
+)
+
+func newHealthEndpoint(e *Engine) {
+	e.Router.Get("/health", func(w http.ResponseWriter, _ *http.Request) {
+		SafeWrite(w.Write, []byte("OK"))
+	})
+}

internal/engine/resources.go

@@ -9,19 +9,23 @@ import (
 	"github.com/go-chi/chi/v5"
 )
 
-// Serve static assets either from local storage or through reverse proxy
+// Resources endpoint to serve static assets, either from local storage or through reverse proxy
 func newResourcesEndpoint(e *Engine) {
-	if e.Config.Resources.Directory != nil && *e.Config.Resources.Directory != "" {
-		resourcesPath := strings.TrimSuffix(*e.Config.Resources.Directory, "/resources")
+	res := e.Config.Resources
+	if res == nil {
+		return
+	}
+	if res.Directory != nil && *res.Directory != "" {
+		resourcesPath := strings.TrimSuffix(*res.Directory, "/resources")
 		e.Router.Handle("/resources/*", http.FileServer(http.Dir(resourcesPath)))
-	} else if e.Config.Resources.URL != nil && e.Config.Resources.URL.String() != "" {
-		e.Router.Get("/resources/*", proxy(e.ReverseProxy, e.Config.Resources.URL.String()))
+	} else if res.URL != nil && res.URL.String() != "" {
+		e.Router.Get("/resources/*", proxy(e.ReverseProxy, res.URL.String()))
 	}
 }
 
-type reverseProxy func(w http.ResponseWriter, r *http.Request, target *url.URL, prefer204 bool, overwrite string)
+type revProxy func(w http.ResponseWriter, r *http.Request, target *url.URL, prefer204 bool, overwrite string)
 
-func proxy(rp reverseProxy, resourcesURL string) func(w http.ResponseWriter, r *http.Request) {
+func proxy(revProxy revProxy, resourcesURL string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		resourcePath, _ := url.JoinPath("/", chi.URLParam(r, "*"))
 		target, err := url.ParseRequestURI(resourcesURL + resourcePath)
@@ -30,6 +34,6 @@ func proxy(rp reverseProxy, resourcesURL string) func(w http.ResponseWriter, r *
 			RenderProblem(ProblemServerError, w)
 			return
 		}
-		rp(w, r, target, true, "")
+		revProxy(w, r, target, true, "")
 	}
 }

internal/engine/sitemap.go

@@ -0,0 +1,13 @@
+package engine
+
+import "net/http"
+
+func newSitemap(e *Engine) {
+	for path, template := range map[string]string{"/sitemap.xml": "sitemap.go.xml", "/robots.txt": "robots.go.txt"} {
+		key := NewTemplateKey(templatesDir + template)
+		e.renderTemplates(path, nil, nil, false, key)
+		e.Router.Get(path, func(w http.ResponseWriter, r *http.Request) {
+			e.serve(w, r, &key, false, false, "", nil)
+		})
+	}
+}