api: /censuses/export now streams the response (dirty)

api/censusdb/censusdb.go

@@ -379,6 +379,7 @@ func (c *CensusDB) ExportCensusDB(buffer io.Writer) error {
 	defer c.Unlock()
 	// Iterate through all census entries in the DB
 	err := c.db.Iterate([]byte(censusDBreferencePrefix), func(key, data []byte) bool {
+		log.Warnf("iterate %x (%s), %x (%s)", key, key, data, data)
 		censusID := bytes.Clone(key)
 		dec := gob.NewDecoder(bytes.NewReader(data))
 		ref := CensusRef{}

api/censuses.go

@@ -984,27 +984,33 @@ func (a *API) censusListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext)
 //	@Router			/censuses/export/ipfs [get]
 //	@Router			/censuses/export [get]
 func (a *API) censusExportDBHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error {
-	isIPFSExport := strings.HasSuffix(ctx.Request.URL.Path, "ipfs")
-	buf := bytes.Buffer{}
-	if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+	w, err := ctx.SendStream(apirest.HTTPstatusOK)
+	if err != nil {
 		return err
 	}
-	var data []byte
-	if isIPFSExport {
+	if strings.HasSuffix(ctx.Request.URL.Path, "ipfs") {
+		buf := bytes.Buffer{}
+		log.Warn("start export censusdb") // debug
+		if err := a.censusdb.ExportCensusDB(&buf); err != nil {
+			return err
+		}
+		log.Warn("end export censusdb") // debug
 		uri, err := a.storage.PublishReader(ctx.Request.Context(), &buf)
 		if err != nil {
 			return err
 		}
-		data, err = json.Marshal(map[string]string{
+		data, err := json.Marshal(map[string]string{
 			"uri": uri,
 		})
 		if err != nil {
 			return err
 		}
-	} else {
-		data = buf.Bytes()
+		_, err = w.Write(data)
+		return err
 	}
-	return ctx.Send(data, apirest.HTTPstatusOK)
+	log.Warn("start export censusdb")     // debug
+	defer log.Warn("end export censusdb") // debug
+	return a.censusdb.ExportCensusDB(w)
 }
 
 // censusImportHandler

httprouter/message.go

@@ -2,6 +2,7 @@ package httprouter
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"time"
 	"unicode/utf8"
@@ -114,3 +115,43 @@ func (h *HTTPContext) Send(msg []byte, httpStatusCode int) error {
 	_, err := h.writer.Write([]byte("\n"))
 	return err
 }
+
+// SendStream replies the request with the status code and returns a io.Writer so the data can be written into it.
+func (h *HTTPContext) SendStream(httpStatusCode int) (io.Writer, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warnf("recovered http send panic: %v", r)
+		}
+	}()
+	defer close(h.sent)
+
+	if httpStatusCode < 100 || httpStatusCode >= 600 {
+		return nil, fmt.Errorf("http status code %d not supported", httpStatusCode)
+	}
+	if h.Request.Context().Err() != nil {
+		// The connection was closed, so don't try to write to it.
+		return nil, fmt.Errorf("connection is closed")
+	}
+	// Set the content type if not set to default application/json
+	if h.contentType == "" {
+		h.writer.Header().Set("Content-Type", DefaultContentType)
+	} else {
+		h.writer.Header().Set("Content-Type", h.contentType)
+	}
+
+	// Special handling for no content, reset content, and not modified.
+	if httpStatusCode == http.StatusNoContent ||
+		httpStatusCode == http.StatusResetContent ||
+		httpStatusCode == http.StatusNotModified ||
+		(httpStatusCode >= 100 && httpStatusCode < 200) {
+		// Don't set Content-Length, don't try to write a body.
+		h.writer.WriteHeader(httpStatusCode)
+		log.Debugw("http response", "status", httpStatusCode)
+		return nil, nil
+	}
+
+	h.writer.WriteHeader(httpStatusCode)
+
+	log.Debugw("http response will be streamed in chunks", "status", httpStatusCode)
+	return h.writer, nil
+}