From c318727067510bf412baf736c857ca9235bba0a2 Mon Sep 17 00:00:00 2001
From: Gui Iribarren <gui@altermundi.net>
Date: Mon, 16 Sep 2024 18:00:24 +0200
Subject: [PATCH] api: /censuses/export now streams the response (dirty)

---
 api/censusdb/censusdb.go |  1 +
 api/censuses.go          | 24 ++++++++++++++---------
 httprouter/message.go    | 41 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/api/censusdb/censusdb.go b/api/censusdb/censusdb.go
index 6bd63f1ea..66a66fea2 100644
--- a/api/censusdb/censusdb.go
+++ b/api/censusdb/censusdb.go
@@ -379,6 +379,7 @@ func (c *CensusDB) ExportCensusDB(buffer io.Writer) error {
 	defer c.Unlock()
 	// Iterate through all census entries in the DB
 	err := c.db.Iterate([]byte(censusDBreferencePrefix), func(key, data []byte) bool {
+		log.Warnf("iterate %x (%s), %x (%s)", key, key, data, data)
 		censusID := bytes.Clone(key)
 		dec := gob.NewDecoder(bytes.NewReader(data))
 		ref := CensusRef{}
diff --git a/api/censuses.go b/api/censuses.go
index 31ec08c7c..1a82ed428 100644
--- a/api/censuses.go
+++ b/api/censuses.go
@@ -984,27 +984,33 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 //	@Router			/censuses/export/ipfs [get]
 //	@Router			/censuses/export [get]
 func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
-	isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs")
-	buf := bytes.Buffer{}
-	if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+	w, err := ctx.SendStream(apirest.HTTPstatusOK)
+	if err != nil {
 		return err
 	}
-	var data []byte
-	if isIPFSExport {
+	if strings.HasSuffix(ctx.Request.URL.Path, "ipfs") {
+		buf := bytes.Buffer{}
+		log.Warn("start export censusdb") // debug
+		if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+			return err
+		}
+		log.Warn("end export censusdb") // debug
 		uri, err := a.storage.PublishReader(ctx.Request.Context(), &buf)
 		if err != nil {
 			return err
 		}
-		data, err = json.Marshal(map[string]string{
+		data, err := json.Marshal(map[string]string{
 			"uri": uri,
 		})
 		if err != nil {
 			return err
 		}
-	} else {
-		data = buf.Bytes()
+		_, err = w.Write(data)
+		return err
 	}
-	return ctx.Send(data, apirest.HTTPstatusOK)
+	log.Warn("start export censusdb")     // debug
+	defer log.Warn("end export censusdb") // debug
+	return a.censusdb.ExportCensusDB(w)
 }
 
 // censusImportHandler
diff --git a/httprouter/message.go b/httprouter/message.go
index b1723dd29..7757ee6af 100644
--- a/httprouter/message.go
+++ b/httprouter/message.go
@@ -2,6 +2,7 @@ package httprouter
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 	"unicode/utf8"
@@ -114,3 +115,43 @@ func (h *HTTPContext) Send(msg []byte, httpStatusCode int) error {
 	_, err := h.writer.Write([]byte("\n"))
 	return err
 }
+
+// SendStream replies the request with the status code and returns a io.Writer so the data can be written into it.
+func (h *HTTPContext) SendStream(httpStatusCode int) (io.Writer, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warnf("recovered http send panic: %v", r)
+		}
+	}()
+	defer close(h.sent)
+
+	if httpStatusCode < 100 || httpStatusCode >= 600 {
+		return nil, fmt.Errorf("http status code %d not supported", httpStatusCode)
+	}
+	if h.Request.Context().Err() != nil {
+		// The connection was closed, so don't try to write to it.
+		return nil, fmt.Errorf("connection is closed")
+	}
+	// Set the content type if not set to default application/json
+	if h.contentType == "" {
+		h.writer.Header().Set("Content-Type", DefaultContentType)
+	} else {
+		h.writer.Header().Set("Content-Type", h.contentType)
+	}
+
+	// Special handling for no content, reset content, and not modified.
+	if httpStatusCode == http.StatusNoContent ||
+		httpStatusCode == http.StatusResetContent ||
+		httpStatusCode == http.StatusNotModified ||
+		(httpStatusCode >= 100 && httpStatusCode < 200) {
+		// Don't set Content-Length, don't try to write a body.
+		h.writer.WriteHeader(httpStatusCode)
+		log.Debugw("http response", "status", httpStatusCode)
+		return nil, nil
+	}
+
+	h.writer.WriteHeader(httpStatusCode)
+
+	log.Debugw("http response will be streamed in chunks", "status", httpStatusCode)
+	return h.writer, nil
+}