From d4f79c33d8d98b43cdb8b0336c8e81ac07488a95 Mon Sep 17 00:00:00 2001 From: Richard Kettelerij Date: Thu, 26 Sep 2024 16:22:28 +0200 Subject: [PATCH] feat: setup minimal sitemap.xml + robots.txt. Also made the code more DRY in order to reuse rendering funcs for the sitemap. --- README.md | 4 +- hack/setup-jetbrains-gotemplates.sh | 5 + internal/engine/contentnegotiation.go | 4 + internal/engine/contentnegotiation_test.go | 3 +- internal/engine/engine.go | 125 +++++++++------------ internal/engine/health.go | 11 ++ internal/engine/resources.go | 20 ++-- internal/engine/sitemap.go | 13 +++ internal/engine/templates/robots.go.txt | 4 + internal/engine/templates/sitemap.go.xml | 7 ++ internal/ogc/common/core/main.go | 2 +- internal/ogc/features/json.go | 2 +- 12 files changed, 116 insertions(+), 84 deletions(-) create mode 100644 internal/engine/health.go create mode 100644 internal/engine/sitemap.go create mode 100644 internal/engine/templates/robots.go.txt create mode 100644 internal/engine/templates/sitemap.go.xml diff --git a/README.md b/README.md index aaf1c70b..4b02ca92 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ already being taken care of when building the Docker container image. - `"*.go.html"` - `"*.go.json"` - `"*.go.tilejson"` + - `"*.go.xml"` - Now add template language support by running the [setup-jetbrains-gotemplates.sh](hack/setup-jetbrains-gotemplates.sh) script. - Reopen the project (or restart IDE). Now you'll have full IDE support in the GoKoala templates. @@ -190,7 +191,8 @@ Also: - `"*.go.html"` - `"*.go.json"` - `"*.go.tilejson"` -- Also add `html` and `json` to the list of Go template languages. + - `"*.go.xml"` +- Also add `html`, `json` and `xml` to the list of Go template languages. - Now you'll have IDE support in the GoKoala templates. ### OGC compliance validation diff --git a/hack/setup-jetbrains-gotemplates.sh b/hack/setup-jetbrains-gotemplates.sh index 4389fb1a..d8e9af89 100755 --- a/hack/setup-jetbrains-gotemplates.sh +++ b/hack/setup-jetbrains-gotemplates.sh @@ -23,6 +23,11 @@ traverse_templates() { echo "Processing file: $file" echo "" >> ".idea/templateLanguages.xml" done + # XML templates + find * -type f -iname "*.go.xml" -print0 | while IFS= read -r -d '' file; do + echo "Processing file: $file" + echo "" >> ".idea/templateLanguages.xml" + done } mkdir -p ".idea/" diff --git a/internal/engine/contentnegotiation.go b/internal/engine/contentnegotiation.go index cdba16d2..881d638c 100644 --- a/internal/engine/contentnegotiation.go +++ b/internal/engine/contentnegotiation.go @@ -15,6 +15,7 @@ const ( languageParam = "lang" MediaTypeJSON = "application/json" + MediaTypeXML = "application/xml" MediaTypeHTML = "text/html" MediaTypeTileJSON = "application/vnd.mapbox.tile+json" MediaTypeMVT = "application/vnd.mapbox-vector-tile" @@ -26,6 +27,7 @@ const ( MediaTypeQuantizedMesh = "application/vnd.quantized-mesh" FormatHTML = "html" + FormatXML = "xml" FormatJSON = "json" FormatTileJSON = "tilejson" FormatMVT = "mvt" @@ -74,6 +76,7 @@ func newContentNegotiation(availableLanguages []config.Language) *ContentNegotia availableMediaTypes := []contenttype.MediaType{ // in order contenttype.NewMediaType(MediaTypeJSON), + contenttype.NewMediaType(MediaTypeXML), contenttype.NewMediaType(MediaTypeHTML), contenttype.NewMediaType(MediaTypeTileJSON), contenttype.NewMediaType(MediaTypeGeoJSON), @@ -85,6 +88,7 @@ func newContentNegotiation(availableLanguages []config.Language) *ContentNegotia formatsByMediaType := map[string]string{ MediaTypeJSON: FormatJSON, + MediaTypeXML: FormatXML, MediaTypeHTML: FormatHTML, MediaTypeTileJSON: FormatTileJSON, MediaTypeGeoJSON: FormatGeoJSON, diff --git a/internal/engine/contentnegotiation_test.go b/internal/engine/contentnegotiation_test.go index 4c29952c..947128d9 100644 --- a/internal/engine/contentnegotiation_test.go +++ b/internal/engine/contentnegotiation_test.go @@ -23,7 +23,8 @@ func TestContentNegotiation_NegotiateFormat(t *testing.T) { testFormat(t, cn, "application/json", "http://pdok.example/ogc/api.json", "json") testFormat(t, cn, "application/json", "http://pdok.example/ogc/api?f=json", "json") testFormat(t, cn, "", "http://pdok.example/ogc/api?f=json", "json") - testFormat(t, cn, "application/xml, application/json, text/css, text/html", "http://pdok.example/ogc/api/", "json") + testFormat(t, cn, "application/xml, application/json, text/css, text/html", "http://pdok.example/ogc/api/", "xml") + testFormat(t, cn, "application/json, application/xml, text/css, text/html", "http://pdok.example/ogc/api/", "json") testLanguage(t, cn, "nl;q=1", "http://pdok.example/ogc/api", language.Dutch) testLanguage(t, cn, "fr;q=0.8, de;q=0.5", "http://pdok.example/ogc/api", language.Dutch) testLanguage(t, cn, "en;q=1", "http://pdok.example/ogc/api", language.English) diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 20f92984..bd64f5a4 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -77,12 +77,10 @@ func NewEngineWithConfig(config *config.Config, openAPIFile string, enableTraili Router: router, } - if config.Resources != nil { - newResourcesEndpoint(engine) // Resources endpoint to serve static assets - } - router.Get("/health", func(w http.ResponseWriter, _ *http.Request) { - SafeWrite(w.Write, []byte("OK")) // Health endpoint - }) + // Default (non-OGC) endpoints + newSitemap(engine) + newHealthEndpoint(engine) + newResourcesEndpoint(engine) return engine } @@ -172,42 +170,37 @@ func (e *Engine) ParseTemplate(key TemplateKey) { // This method also performs OpenAPI validation of the rendered template, therefore we also need the URL path. // The rendered templates are stored in the engine for future serving using ServePage. func (e *Engine) RenderTemplates(urlPath string, breadcrumbs []Breadcrumb, keys ...TemplateKey) { - for _, key := range keys { - e.Templates.renderAndSaveTemplate(key, breadcrumbs, nil) - - // we already perform OpenAPI validation here during startup to catch - // issues early on, in addition to runtime OpenAPI response validation - // all templates are created in all available languages, hence all are checked - for lang := range e.Templates.localizers { - key.Language = lang - if err := e.validateStaticResponse(key, urlPath); err != nil { - log.Fatal(err) - } - } - } + e.renderTemplates(urlPath, nil, breadcrumbs, true, keys...) } // RenderTemplatesWithParams renders both HTMl and non-HTML templates depending on the format given in the TemplateKey. func (e *Engine) RenderTemplatesWithParams(urlPath string, params any, breadcrumbs []Breadcrumb, keys ...TemplateKey) { + e.renderTemplates(urlPath, params, breadcrumbs, true, keys...) +} + +func (e *Engine) renderTemplates(urlPath string, params any, breadcrumbs []Breadcrumb, validate bool, keys ...TemplateKey) { for _, key := range keys { e.Templates.renderAndSaveTemplate(key, breadcrumbs, params) - // we already perform OpenAPI validation here during startup to catch - // issues early on, in addition to runtime OpenAPI response validation - // all templates are created in all available languages, hence all are checked - for lang := range e.Templates.localizers { - key.Language = lang - if err := e.validateStaticResponse(key, urlPath); err != nil { - log.Fatal(err) + if validate { + // we already perform OpenAPI validation here during startup to catch + // issues early on, in addition to runtime OpenAPI response validation + // all templates are created in all available languages, hence all are checked + for lang := range e.Templates.localizers { + key.Language = lang + if err := e.validateStaticResponse(key, urlPath); err != nil { + log.Fatal(err) + } } } } } -// RenderAndServePage renders an already parsed HTML or non-HTML template and renders it on-the-fly depending +// RenderAndServePage renders an already parsed HTML or non-HTML template on-the-fly depending // on the format in the given TemplateKey. The result isn't store in engine, it's served directly to the client. // -// NOTE: only used this for dynamic pages that can't be pre-rendered and cached (e.g. with data from a backing store). +// NOTE: only used this for dynamic pages that can't be pre-rendered and cached (e.g. with data from a datastore), +// otherwise use ServePage for pre-rendered pages. func (e *Engine) RenderAndServePage(w http.ResponseWriter, r *http.Request, key TemplateKey, params any, breadcrumbs []Breadcrumb) { @@ -242,49 +235,22 @@ func (e *Engine) RenderAndServePage(w http.ResponseWriter, r *http.Request, key RenderProblem(ProblemServerError, w, err.Error()) return } - - // return response output to client - if contentType != "" { - w.Header().Set(HeaderContentType, contentType) - } - SafeWrite(w.Write, output) + writeResponse(w, contentType, output) } // ServePage serves a pre-rendered template while also validating against the OpenAPI spec func (e *Engine) ServePage(w http.ResponseWriter, r *http.Request, templateKey TemplateKey) { - // validate request - if err := e.OpenAPI.ValidateRequest(r); err != nil { - log.Printf("%v", err.Error()) - RenderProblem(ProblemBadRequest, w, err.Error()) - return - } - - // render output - output, err := e.Templates.getRenderedTemplate(templateKey) - if err != nil { - log.Printf("%v", err.Error()) - RenderProblem(ProblemNotFound, w) - return - } - contentType := e.CN.formatToMediaType(templateKey.Format) - - // validate response - if err := e.OpenAPI.ValidateResponse(contentType, output, r); err != nil { - log.Printf("%v", err.Error()) - RenderProblem(ProblemServerError, w, err.Error()) - return - } + e.serve(w, r, &templateKey, true, true, "", nil) +} - // return response output to client - if contentType != "" { - w.Header().Set(HeaderContentType, contentType) - } - SafeWrite(w.Write, output) +// Serve serves the given response (arbitrary bytes) while also validating against the OpenAPI spec +func (e *Engine) Serve(w http.ResponseWriter, r *http.Request, + validateRequest bool, validateResponse bool, contentType string, output []byte) { + e.serve(w, r, nil, validateRequest, validateResponse, contentType, output) } -// ServeResponse serves the given response (arbitrary bytes) while also validating against the OpenAPI spec -func (e *Engine) ServeResponse(w http.ResponseWriter, r *http.Request, - validateRequest bool, validateResponse bool, contentType string, response []byte) { +func (e *Engine) serve(w http.ResponseWriter, r *http.Request, templateKey *TemplateKey, + validateRequest bool, validateResponse bool, contentType string, output []byte) { if validateRequest { if err := e.OpenAPI.ValidateRequest(r); err != nil { @@ -294,19 +260,26 @@ func (e *Engine) ServeResponse(w http.ResponseWriter, r *http.Request, } } - if validateResponse { - if err := e.OpenAPI.ValidateResponse(contentType, response, r); err != nil { + if templateKey != nil { + // render output + var err error + output, err = e.Templates.getRenderedTemplate(*templateKey) + if err != nil { log.Printf("%v", err.Error()) - RenderProblem(ProblemServerError, w, err.Error()) + RenderProblem(ProblemNotFound, w) return } + contentType = e.CN.formatToMediaType(templateKey.Format) } - // return response output to client - if contentType != "" { - w.Header().Set(HeaderContentType, contentType) + if validateResponse { + if err := e.OpenAPI.ValidateResponse(contentType, output, r); err != nil { + log.Printf("%v", err.Error()) + RenderProblem(ProblemServerError, w, err.Error()) + return + } } - SafeWrite(w.Write, response) + writeResponse(w, contentType, output) } // ReverseProxy forwards given HTTP request to given target server, and optionally tweaks response @@ -359,7 +332,7 @@ func (e *Engine) ReverseProxyAndValidate(w http.ResponseWriter, r *http.Request, if err != nil { return err } - e.ServeResponse(w, r, false, true, contentType, res) + e.Serve(w, r, false, true, contentType, res) } return nil } @@ -394,6 +367,14 @@ func (e *Engine) validateStaticResponse(key TemplateKey, urlPath string) error { return nil } +// return response output to client +func writeResponse(w http.ResponseWriter, contentType string, output []byte) { + if contentType != "" { + w.Header().Set(HeaderContentType, contentType) + } + SafeWrite(w.Write, output) +} + // SafeWrite executes the given http.ResponseWriter.Write while logging errors func SafeWrite(write func([]byte) (int, error), body []byte) { _, err := write(body) diff --git a/internal/engine/health.go b/internal/engine/health.go new file mode 100644 index 00000000..17097824 --- /dev/null +++ b/internal/engine/health.go @@ -0,0 +1,11 @@ +package engine + +import ( + "net/http" +) + +func newHealthEndpoint(e *Engine) { + e.Router.Get("/health", func(w http.ResponseWriter, _ *http.Request) { + SafeWrite(w.Write, []byte("OK")) + }) +} diff --git a/internal/engine/resources.go b/internal/engine/resources.go index 09691c5b..55bfef4a 100644 --- a/internal/engine/resources.go +++ b/internal/engine/resources.go @@ -9,19 +9,23 @@ import ( "github.com/go-chi/chi/v5" ) -// Serve static assets either from local storage or through reverse proxy +// Resources endpoint to serve static assets, either from local storage or through reverse proxy func newResourcesEndpoint(e *Engine) { - if e.Config.Resources.Directory != nil && *e.Config.Resources.Directory != "" { - resourcesPath := strings.TrimSuffix(*e.Config.Resources.Directory, "/resources") + res := e.Config.Resources + if res == nil { + return + } + if res.Directory != nil && *res.Directory != "" { + resourcesPath := strings.TrimSuffix(*res.Directory, "/resources") e.Router.Handle("/resources/*", http.FileServer(http.Dir(resourcesPath))) - } else if e.Config.Resources.URL != nil && e.Config.Resources.URL.String() != "" { - e.Router.Get("/resources/*", proxy(e.ReverseProxy, e.Config.Resources.URL.String())) + } else if res.URL != nil && res.URL.String() != "" { + e.Router.Get("/resources/*", proxy(e.ReverseProxy, res.URL.String())) } } -type reverseProxy func(w http.ResponseWriter, r *http.Request, target *url.URL, prefer204 bool, overwrite string) +type revProxy func(w http.ResponseWriter, r *http.Request, target *url.URL, prefer204 bool, overwrite string) -func proxy(rp reverseProxy, resourcesURL string) func(w http.ResponseWriter, r *http.Request) { +func proxy(revProxy revProxy, resourcesURL string) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { resourcePath, _ := url.JoinPath("/", chi.URLParam(r, "*")) target, err := url.ParseRequestURI(resourcesURL + resourcePath) @@ -30,6 +34,6 @@ func proxy(rp reverseProxy, resourcesURL string) func(w http.ResponseWriter, r * RenderProblem(ProblemServerError, w) return } - rp(w, r, target, true, "") + revProxy(w, r, target, true, "") } } diff --git a/internal/engine/sitemap.go b/internal/engine/sitemap.go new file mode 100644 index 00000000..74a86ad1 --- /dev/null +++ b/internal/engine/sitemap.go @@ -0,0 +1,13 @@ +package engine + +import "net/http" + +func newSitemap(e *Engine) { + for path, template := range map[string]string{"/sitemap.xml": "sitemap.go.xml", "/robots.txt": "robots.go.txt"} { + key := NewTemplateKey(templatesDir + template) + e.renderTemplates(path, nil, nil, false, key) + e.Router.Get(path, func(w http.ResponseWriter, r *http.Request) { + e.serve(w, r, &key, false, false, "", nil) + }) + } +} diff --git a/internal/engine/templates/robots.go.txt b/internal/engine/templates/robots.go.txt new file mode 100644 index 00000000..88666ca6 --- /dev/null +++ b/internal/engine/templates/robots.go.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: {{ .Config.BaseURL }}/sitemap.xml \ No newline at end of file diff --git a/internal/engine/templates/sitemap.go.xml b/internal/engine/templates/sitemap.go.xml new file mode 100644 index 00000000..82149fde --- /dev/null +++ b/internal/engine/templates/sitemap.go.xml @@ -0,0 +1,7 @@ + + + + https: //www.example.com/foo.html + 2022-06-04 + + \ No newline at end of file diff --git a/internal/ogc/common/core/main.go b/internal/ogc/common/core/main.go index fd8a3f99..1ebbdb28 100644 --- a/internal/ogc/common/core/main.go +++ b/internal/ogc/common/core/main.go @@ -84,7 +84,7 @@ func (c *CommonCore) apiAsHTML(w http.ResponseWriter, r *http.Request) { } func (c *CommonCore) apiAsJSON(w http.ResponseWriter, r *http.Request) { - c.engine.ServeResponse(w, r, true, true, engine.MediaTypeOpenAPI, c.engine.OpenAPI.SpecJSON) + c.engine.Serve(w, r, true, true, engine.MediaTypeOpenAPI, c.engine.OpenAPI.SpecJSON) } func (c *CommonCore) Conformance() http.HandlerFunc { diff --git a/internal/ogc/features/json.go b/internal/ogc/features/json.go index 763ef560..dc2f9645 100644 --- a/internal/ogc/features/json.go +++ b/internal/ogc/features/json.go @@ -301,7 +301,7 @@ func (jf *jsonFeatures) serveAndValidateJSON(input any, contentType string, r *h handleJSONEncodingFailure(err, w) return } - jf.engine.ServeResponse(w, r, false /* performed earlier */, jf.validateResponse, contentType, json.Bytes()) + jf.engine.Serve(w, r, false /* performed earlier */, jf.validateResponse, contentType, json.Bytes()) } // serveJSON serves JSON *WITHOUT* OpenAPI validation by writing directly to the response output stream