New server side endpoint for getting (all) mutations, which provides …

…data in chunks (#617)

* mutation chunks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* exclude helm charts from prettier

* adjust prettier exclusion

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.prettierignore

@@ -0,0 +1 @@
+helm_charts/**

helm_charts/taxonium-backend/templates/updater.yaml

@@ -42,4 +42,4 @@ subjects:
 roleRef:
   kind: Role
   name: deployment-restarter
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

taxonium_backend/server.js

@@ -13,6 +13,7 @@ var pako = require("pako");
 const URL = require("url").URL;
 const ReadableWebToNodeStream = require("readable-web-to-node-stream");
 const { execSync } = require("child_process");
+const { Readable } = require("stream");
 var importing;
 var filtering;
 var exporting;
@@ -206,14 +207,54 @@ app.get("/config", function (req, res) {
     (processedData.overallMinY + processedData.overallMaxY) / 2;
   config.initial_zoom = -2;
   config.genes = processedData.genes;
-  config.mutations = processedData.mutations;
   config = { ...config, ...processedData.overwrite_config };
   config.rootMutations = processedData.rootMutations;
   config.rootId = processedData.rootId;
 
   res.send(config);
 });
 
+app.get("/mutations/", function (req, res) {
+  // Set headers for SSE
+  res.writeHead(200, {
+    "Content-Type": "text/event-stream",
+    "Cache-Control": "no-cache",
+    Connection: "keep-alive",
+  });
+
+  // Function to send SSE
+  function sendSSE(data) {
+    res.write(`data: ${data}\n\n`);
+  }
+
+  // Send mutations in chunks of 1000
+  const chunkSize = 10000;
+  let index = 0;
+
+  function sendNextChunk() {
+    const chunk = processedData.mutations.slice(index, index + chunkSize);
+    if (chunk.length > 0) {
+      sendSSE(JSON.stringify(chunk));
+      index += chunkSize;
+      // Schedule the next chunk
+      setImmediate(sendNextChunk);
+    } else {
+      // All mutations sent, end the stream
+      sendSSE("END");
+      res.end();
+    }
+  }
+
+  // Start sending chunks
+  sendNextChunk();
+
+  // Handle client disconnect
+  req.on("close", () => {
+    // No need to destroy a stream, just stop the process
+    index = processedData.mutations.length; // This will stop sendNextChunk on next iteration
+  });
+});
+
 app.get("/nodes/", function (req, res) {
   const start_time = Date.now();
   let min_y =

taxonium_component/src/hooks/useServerBackend.jsx

@@ -121,20 +121,69 @@ function useServerBackend(backend_url, sid, url_on_fail) {
     },
     [backend_url, sid]
   );
-
   const getConfig = useCallback(
     (setResult) => {
-      let url = backend_url + "/config/?sid=" + sid;
-      axios.get(url).then(function (response) {
-        console.log("got config", response.data);
-        if (response.data.error) {
-          window.alert(
-            response.data.error + (url_on_fail ? "\nRedirecting you." : "")
-          );
-          window.location.href = url_on_fail;
-        }
-        setResult(response.data);
-      });
+      const url = `${backend_url}/config/?sid=${sid}`;
+
+      // Fetch initial config
+      axios
+        .get(url)
+        .then((response) => {
+          console.log("got config", response.data);
+          if (response.data.error) {
+            window.alert(
+              response.data.error + (url_on_fail ? "\nRedirecting you." : "")
+            );
+            window.location.href = url_on_fail;
+            return;
+          }
+
+          const config = response.data;
+          config.mutations = config.mutations ? config.mutations : [];
+
+          // Stream mutations
+          const mutationsUrl = `${backend_url}/mutations/?sid=${sid}`;
+          const eventSource = new EventSource(mutationsUrl);
+
+          eventSource.onmessage = (event) => {
+            if (event.data === "END") {
+              console.log("Finished receiving mutations");
+              eventSource.close();
+              setResult(config);
+              return;
+            }
+
+            try {
+              const mutationsChunk = JSON.parse(event.data);
+              if (Array.isArray(mutationsChunk)) {
+                config.mutations.push(...mutationsChunk);
+                setResult({ ...config });
+                console.log(
+                  `Received chunk of ${mutationsChunk.length} mutations`
+                );
+              } else {
+                console.error("Received non-array chunk:", mutationsChunk);
+              }
+            } catch (error) {
+              console.error("Error parsing mutations chunk:", error);
+            }
+          };
+
+          eventSource.onerror = (error) => {
+            console.error("EventSource failed:", error);
+            eventSource.close();
+          };
+
+          // Set initial config
+          setResult(config);
+        })
+        .catch((error) => {
+          console.error("Error fetching config:", error);
+          if (url_on_fail) {
+            window.alert("Failed to fetch config. Redirecting you.");
+            window.location.href = url_on_fail;
+          }
+        });
     },
     [backend_url, sid, url_on_fail]
   );