feat: add an optional stats panel to MediapipeCamera

cdiddy77 · Apr 1, 2024 · aa01b6b · aa01b6b
1 parent 4e2be6e
commit aa01b6b
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 22 deletions.
diff --git a/src/index.tsx b/src/index.tsx
@@ -12,10 +12,12 @@ type MediapipeProps = {
   resultsPanel: boolean;
 };
 
-// TODONEXT: Figure out how to get the callback to work
-//  List the objects detected in the results panel
-//  Make that panel be scrollable
+// TODO:
+//  Keep track of the last non-zero infereance and display that for
+//   some time after the inference is done if the next ones are zero
+//  Make the stats panel scrollable
 //  Clean up the layout
+//  displayName is empty - should this be filled in by the frameProcessor?
 export const MediapipeCamera: React.FC<MediapipeProps> = ({
   style,
   resultsPanel,
@@ -25,8 +27,6 @@ export const MediapipeCamera: React.FC<MediapipeProps> = ({
   const frameProcessor = useObjectDetection(
     (results) => {
       setStats(results);
-      console.log("WE'VE GOT RESULTS");
-      console.log(results);
     },
     (error) => {
       console.log(error);
@@ -49,6 +49,25 @@ export const MediapipeCamera: React.FC<MediapipeProps> = ({
     />
   );
 
+  let rows: React.JSX.Element[] = [];
+  let count = 0;
+
+  if ((stats?.results.length ?? 0) > 1) {
+    console.log("more than one result", stats?.results.length);
+  }
+
+  if (stats?.results && stats.results.length > 0 && stats.results[0]) {
+    count = stats.results[0].detections.length;
+    rows = stats.results[0].detections.map((obj) => {
+      const categories = obj.categories
+        .map((cat) => `${cat.categoryName} (${cat.score})`)
+        .join(", ");
+      const text = `${categories}: ${JSON.stringify(obj.boundingBox)}`;
+      console.log(text);
+      return <Text>{text}</Text>;
+    });
+  }
+
   return resultsPanel ? (
     <View style={{ flexDirection: "column", alignItems: "stretch" }}>
       {camera}
@@ -61,8 +80,9 @@ export const MediapipeCamera: React.FC<MediapipeProps> = ({
         }}
       >
         <View style={{ width: "100%" }}>
-          <Text>count: {stats?.results.length}</Text>
           <Text>inferenceTime: {stats?.inferenceTime}</Text>
+          <Text>count: {count}</Text>
+          {rows}
         </View>
       </View>
     </View>

diff --git a/src/objectDetection/index.ts b/src/objectDetection/index.ts
@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useState } from "react";
 import { NativeEventEmitter, NativeModules } from "react-native";
 import {
   VisionCameraProxy,
@@ -104,11 +104,6 @@ eventEmitter.addListener(
     //console.log("onResults", JSON.stringify(args));
     const callbacks = detectorMap.get(args.handle);
     if (callbacks) {
-      // TODO: Looks like this isn't getting called -
-      //  when useObjectDetection is registered, we're getting
-      //  `useObjectDetection {"processor": undefined}` so
-      //  we're not finding the callback in the map (I think)
-      console.log("onResults - callback", JSON.stringify(args));
       callbacks.onResults(args);
     }
   }
@@ -130,18 +125,21 @@ export function useObjectDetection(
   model: string,
   options?: Partial<ObjectDetectionOptions>
 ) {
-  console.log("useObjectDetection", { runningMode, model, options });
+  //console.log("useObjectDetection", { runningMode, model, options });
+  const [detectorHandle, setDetectorHandle] = useState<number | undefined>(
+    undefined
+  );
   const processor = useSharedValue<
     | { detectorHandle: number; plugin: FrameProcessorPlugin | undefined }
     | undefined
   >(undefined);
 
   // Remember the latest callback if it changes.
   React.useLayoutEffect(() => {
-    if (processor.value?.detectorHandle !== undefined) {
-      detectorMap.set(processor.value.detectorHandle, { onResults, onError });
+    if (detectorHandle !== undefined) {
+      detectorMap.set(detectorHandle, { onResults, onError });
     }
-  }, [onResults, onError, processor.value?.detectorHandle]);
+  }, [onResults, onError, detectorHandle]);
 
   React.useEffect(() => {
     const plugin =
@@ -158,24 +156,24 @@ export function useObjectDetection(
       .then((handle) => {
         console.log("useObjectDetection", runningMode, model, handle);
         processor.value = { detectorHandle: handle, plugin };
+        setDetectorHandle(handle);
       });
     return () => {
       console.log("useObjectDetection.useEffect.unsub", "releaseDetector");
-      if (processor.value?.detectorHandle !== undefined) {
-        getObjectDetectionModule().releaseDetector(
-          processor.value.detectorHandle
-        );
+      if (detectorHandle !== undefined) {
+        getObjectDetectionModule().releaseDetector(detectorHandle);
       }
     };
+    // The linter wants `processor` and `detectorHandle` in the dependency array, but these are
+    //  _set_ by this useEffect. Making this reactive based on these values would cause an infinite loop
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [
     options?.delegate,
     options?.maxResults,
     runningMode,
     options?.threshold,
-    processor,
     model,
   ]);
-  console.log("useObjectDetection", { processor: processor.value });
   const frameProcessor = useFrameProcessor(
     (frame) => {
       "worklet";