Merge pull request #115 from visdesignlab/112-zstd-decompression

beta-0.9

apps/client/package.json

@@ -49,6 +49,7 @@
         "lru-cache": "^10.2.0",
         "lz-string": "^1.4.4",
         "mitt": "^3.0.1",
+        "numcodecs": "^0.3.2",
         "papaparse": "^5.3.2",
         "pinia": "^2.0.28",
         "quasar": "^2.11.5",
@@ -85,7 +86,7 @@
         "prettier": "^2.7.1",
         "sass": "^1.32.12",
         "typescript": "~4.7.4",
-        "vite": "^4.0.5",
+        "vite": "^5.4.6",
         "vue-tsc": "^1.0.12"
     }
 }

apps/client/src/components/AggregateLineChart.vue

@@ -32,7 +32,7 @@ import {
     TextLayer,
 } from '@deck.gl/layers/typed';
 import type { PixelData, PixelSource } from '@vivjs/types';
-import { Pool } from 'geotiff';
+import Pool from '../util/Pool';
 import {
     loadOmeTiff,
     getChannelStats,

apps/client/src/components/CellTrackView.vue

@@ -16,7 +16,7 @@ import { useImageViewerStoreUntrracked } from '@/stores/imageViewerStoreUntrrack
 import { useDatasetSelectionStore } from '@/stores/datasetSelectionStore';
 import { useEventBusStore } from '@/stores/eventBusStore';
 import { clamp } from 'lodash-es';
-import { Pool } from 'geotiff';
+import Pool from '../util/Pool';
 import type { Feature } from 'geojson';
 import {
     expandHeight,

apps/client/src/components/ImageViewer.vue

@@ -17,7 +17,7 @@ import { useDataPointSelectionUntrracked } from '@/stores/dataPointSelectionUntr
 import { useSegmentationStore } from '@/stores/segmentationStore';
 import { useEventBusStore } from '@/stores/eventBusStore';
 import { clamp } from 'lodash-es';
-import { Pool } from 'geotiff';
+import Pool from '../util/Pool';
 import { useLooneageViewStore } from '@/stores/looneageViewStore';
 import { useGlobalSettings } from '@/stores/globalSettings';
 

apps/client/src/components/LooneageViewGL.vue

@@ -26,7 +26,7 @@ import {
 import { isEqual } from 'lodash-es';
 import { useGlobalSettings } from '@/stores/globalSettings';
 
-import { Pool } from 'geotiff';
+import Pool from '../util/Pool';
 import type { Feature } from 'geojson';
 import { flextree, type LayoutNode } from 'd3-flextree';
 import { hierarchy } from 'd3-hierarchy';

apps/client/src/declarations.d.ts

@@ -0,0 +1 @@
+declare module 'numcodecs';

apps/client/src/util/Pool.ts

@@ -0,0 +1,16 @@
+// Modified from: https://github.com/hms-dbmi/viv
+
+import { Pool } from 'geotiff';
+import DecodeWorker from './decoder.worker?worker';
+
+// https://developer.mozilla.org/en-US/docs/Web/API/NavigatorConcurrentHardware/hardwareConcurrency
+// We need to give a different way of getting this for safari, so 4 is probably a safe bet
+// for parallel processing in the meantime.  More can't really hurt since they'll just block
+// each other and not the UI thread, which is the real benefit.
+const defaultPoolSize = globalThis?.navigator?.hardwareConcurrency ?? 4;
+
+export default class extends Pool {
+    constructor() {
+        super(defaultPoolSize, () => new DecodeWorker());
+    }
+}

apps/client/src/util/decoder.worker.ts

@@ -0,0 +1,37 @@
+// Modified from https://github.com/hms-dbmi/viv
+
+import { getDecoder, addDecoder } from 'geotiff';
+import ZstdDecoder from './zstd-decoder';
+
+addDecoder(50000, () => ZstdDecoder);
+
+// @ts-expect-error - We are in a worker context
+const worker: ServiceWorker = self;
+
+interface WorkerData {
+    id: number;
+    fileDirectory: string;
+    buffer: ArrayBuffer;
+}
+
+async function decodeBuffer(e: MessageEvent<WorkerData>): Promise<void> {
+    try {
+        const { id, fileDirectory, buffer } = e.data;
+        const decoder = await getDecoder(fileDirectory);
+        const decoded = await decoder.decode(fileDirectory, buffer);
+        worker.postMessage({ decoded, id }, [decoded]);
+    } catch (error) {
+        if (error instanceof Error) {
+            console.error(`Failed: ${error.message} at ${e.data.id}`);
+        } else {
+            console.error(`Failed: Generic Error at ${e.data.id}`);
+        }
+    }
+}
+
+worker.addEventListener('message', (event: Event) => {
+    const messageEvent = event as MessageEvent;
+    decodeBuffer(messageEvent).catch((error: Error) => {
+        console.error(`Error during decompression: ${error.message}`);
+    });
+});

apps/client/src/util/zstd-decoder.ts

@@ -0,0 +1,17 @@
+import { BaseDecoder } from 'geotiff';
+import { Zstd } from 'numcodecs';
+
+export default class ZstdDecoder extends BaseDecoder {
+    decoder: any;
+
+    constructor(_: any) {
+        super();
+        this.decoder = new Zstd();
+    }
+
+    async decodeBlock(buffer: ArrayBuffer) {
+        const bytes = new Uint8Array(buffer);
+        const decoded = await this.decoder.decode(bytes);
+        return decoded.buffer;
+    }
+}

apps/client/vite.config.ts

@@ -1,32 +1,33 @@
 import { fileURLToPath, URL } from 'node:url';
-import path from 'path';
-import { defineConfig, loadEnv } from 'vite'
+import { defineConfig, loadEnv } from 'vite';
 
 import vue from '@vitejs/plugin-vue';
 
 import { quasar, transformAssetUrls } from '@quasar/vite-plugin';
 
 export default defineConfig(({ command, mode }) => {
-  // Load env file based on `mode` in the current working directory.
-  // Set the third parameter to '' to load all env regardless of the `VITE_` prefix.
-  const env = loadEnv(mode, process.cwd(), '')
-  return {
-    // vite config
-    plugins: [
-        vue(),
+    // Load env file based on `mode` in the current working directory.
+    // Set the third parameter to '' to load all env regardless of the `VITE_` prefix.
+    const env = loadEnv(mode, process.cwd(), '');
+    return {
+        // vite config
+        plugins: [
+            vue(),
 
-        quasar({
-            sassVariables: 'src/quasar-variables.sass',
-        }),
-    ],
-    resolve: {
-        alias: {
-            '@': fileURLToPath(new URL('./src', import.meta.url)),
+            quasar({
+                sassVariables: 'src/quasar-variables.sass',
+            }),
+        ],
+        resolve: {
+            alias: {
+                '@': fileURLToPath(new URL('./src', import.meta.url)),
+            },
         },
-    },
-    define: {
-        __APP_ENV__: JSON.stringify(env.APP_ENV),
-    },
-  }
-})
-
+        define: {
+            __APP_ENV__: JSON.stringify(env.APP_ENV),
+        },
+        worker: {
+            format: 'es',
+        },
+    };
+});