feat: example app draws rectangles (#28)

.eslintrc.js

@@ -5,7 +5,9 @@ module.exports = {
     tsconfigRootDir: __dirname,
     project: [
       "./tsconfig.json",
-      "docsite/tsconfig.json"
+      "docsite/tsconfig.json",
+      "examples/objectdetection/tsconfig.json",
+      "example/tsconfig.json",
     ],
     ecmaFeatures: {
       jsx: true,
@@ -22,6 +24,8 @@ module.exports = {
     ".prettierrc.js",
     "*.config.js",
     "jest.setup.js",
+    "coverage",
+    "example/index.js",
   ],
   plugins: ["@typescript-eslint"],
   extends: [

.github/workflows/ci.yml

@@ -130,29 +130,20 @@ jobs:
             echo "turbo_cache_hit=1" >> $GITHUB_ENV
           fi
 
-      - name: Generate composite hash
-        id: composite-hash
-        run: |
-          cat example/ios/Podfile.lock examples/objectdetection/ios/Podfile.lock > combined.lock
-          echo "COMPOSITE_HASH=$(sha256sum combined.lock | awk '{print $1}')" >> $GITHUB_ENV
-    
       - name: Cache cocoapods
         if: env.turbo_cache_hit != 1
         id: cocoapods-cache
         uses: actions/cache@v3
         with:
           path: |
             **/ios/Pods
-          key: ${{ runner.os }}-cocoapods-${{ env.COMPOSITE_HASH }}
+          key: ${{ runner.os }}-cocoapods-${{ hashFiles('examples/objectdetection/ios/Podfile.lock') }}
           restore-keys: |
             ${{ runner.os }}-cocoapods-
 
       - name: Install cocoapods
         if: env.turbo_cache_hit != 1 && steps.cocoapods-cache.outputs.cache-hit != 'true'
         run: |
-          cd example/ios
-          pod install
-          cd ../..
           cd examples/objectdetection/ios
           pod install
         env:

android/src/main/java/com/reactnativemediapipe/objectdetection/ObjectDetectorHelper.kt

@@ -13,6 +13,8 @@ import android.renderscript.RenderScript
 import android.renderscript.ScriptIntrinsicYuvToRGB
 import android.renderscript.Type
 import android.util.Log
+import android.os.Handler
+import android.os.Looper
 import androidx.core.math.MathUtils.clamp
 import com.facebook.react.common.annotations.VisibleForTesting
 import com.google.mediapipe.framework.image.BitmapImageBuilder
@@ -48,8 +50,14 @@ class ObjectDetectorHelper(
 
   fun clearObjectDetector() {
     objectDetectorListener = null
-    objectDetector?.close()
-    objectDetector = null
+    // This is a hack. If we call close directly, we crash. There is a theory
+    // that this is because the object detector is still doing some processing, and that
+    // it is not safe to close it. So a better solution might be to mark it and then when
+    // processing is complete, cause it to be closed.
+    Handler(Looper.getMainLooper()).postDelayed({
+      objectDetector?.close()
+      objectDetector = null
+    }, 100)
   }
 
   // Initialize the object detector using current settings on the
@@ -405,7 +413,7 @@ class ObjectDetectorHelper(
     // Convert the input Bitmap object to an MPImage object to run inference
 //    val bitmap = toBitmap(image)
 //    val bitmap = yuv420ToBitmap(image)
-    val bitmap = yuv420ToBitmapRS(image,context)
+    val bitmap = yuv420ToBitmapRS(image, context)
 //    val mpImage = MediaImageBuilder(image).build()
     val mpImage = BitmapImageBuilder(bitmap).build()
 

example/package.json

@@ -8,7 +8,7 @@
     "ios": "react-native run-ios",
     "start": "react-native start",
     "build:android-disabled": "cd android && ./gradlew assembleDebug --no-daemon --console=plain -PreactNativeArchitectures=arm64-v8a",
-    "build:ios": "cd ios && xcodebuild -workspace MediapipeExample.xcworkspace -scheme MediapipeExample -configuration Debug -sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO"
+    "build:ios-disabled": "cd ios && xcodebuild -workspace MediapipeExample.xcworkspace -scheme MediapipeExample -configuration Debug -sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO"
   },
   "dependencies": {
     "react": "18.2.0",
@@ -28,4 +28,4 @@
   "engines": {
     "node": ">=18"
   }
-}
+}

example/src/App.tsx

@@ -60,7 +60,6 @@ const styles = StyleSheet.create({
     flex: 1,
     alignSelf: "stretch",
   },
-  permsButton: {},
   noPermsText: {
     fontSize: 20,
     fontWeight: "bold",

examples/objectdetection/README.md

@@ -1,3 +1,4 @@
+# React Native Mediapipe
 This is a new [**React Native**](https://reactnative.dev) project, bootstrapped using [`@react-native-community/cli`](https://github.com/react-native-community/cli).
 
 # Getting Started

examples/objectdetection/ios/Podfile.lock

@@ -947,6 +947,12 @@ PODS:
   - React-Mapbuffer (0.73.6):
     - glog
     - React-debug
+  - react-native-skia (1.0.5):
+    - glog
+    - RCT-Folly (= 2022.05.16.00)
+    - React
+    - React-callinvoker
+    - React-Core
   - react-native-worklets-core (0.4.0):
     - React
     - React-callinvoker
@@ -1185,6 +1191,7 @@ DEPENDENCIES:
   - React-jsinspector (from `../node_modules/react-native/ReactCommon/jsinspector-modern`)
   - React-logger (from `../node_modules/react-native/ReactCommon/logger`)
   - React-Mapbuffer (from `../node_modules/react-native/ReactCommon`)
+  - "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
   - react-native-worklets-core (from `../node_modules/react-native-worklets-core`)
   - React-nativeconfig (from `../node_modules/react-native/ReactCommon`)
   - React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
@@ -1284,6 +1291,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon/logger"
   React-Mapbuffer:
     :path: "../node_modules/react-native/ReactCommon"
+  react-native-skia:
+    :path: "../node_modules/@shopify/react-native-skia"
   react-native-worklets-core:
     :path: "../node_modules/react-native-worklets-core"
   React-nativeconfig:
@@ -1375,6 +1384,7 @@ SPEC CHECKSUMS:
   React-jsinspector: 85583ef014ce53d731a98c66a0e24496f7a83066
   React-logger: 3eb80a977f0d9669468ef641a5e1fabbc50a09ec
   React-Mapbuffer: 84ea43c6c6232049135b1550b8c60b2faac19fab
+  react-native-skia: 6da30aeaa315fcec5c27ebfb016b9eca2b57b66f
   react-native-worklets-core: 2efe80a3ee87fe5e6fefa814e0e20c2708d3ad25
   React-nativeconfig: b4d4e9901d4cabb57be63053fd2aa6086eb3c85f
   React-NativeModulesApple: cd26e56d56350e123da0c1e3e4c76cb58a05e1ee

examples/objectdetection/package.json

@@ -13,6 +13,7 @@
     "test": "jest"
   },
   "dependencies": {
+    "@shopify/react-native-skia": "^1.0.5",
     "react": "18.2.0",
     "react-native": "0.73.6",
     "react-native-vision-camera": "^3.9.2",

examples/objectdetection/src/App.tsx

@@ -1,13 +1,41 @@
+import {
+  Canvas,
+  Group,
+  Rect,
+  Text as SkiaText,
+  matchFont,
+} from "@shopify/react-native-skia";
 import * as React from "react";
 
-import { Pressable, StyleSheet, Text, View } from "react-native";
-import { MediapipeCamera } from "react-native-mediapipe";
+import {
+  Platform,
+  Pressable,
+  StyleSheet,
+  Text,
+  View,
+  useWindowDimensions,
+} from "react-native";
+import {
+  Delegate,
+  MediapipeCamera,
+  RunningMode,
+  useObjectDetection,
+} from "react-native-mediapipe";
 import {
   useCameraPermission,
   useMicrophonePermission,
 } from "react-native-vision-camera";
 
+interface Detection {
+  label: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
 export default function App(): React.ReactElement | null {
+  const { width, height } = useWindowDimensions();
   const camPerm = useCameraPermission();
   const micPerm = useMicrophonePermission();
   const [permsGranted, setPermsGranted] = React.useState<{
@@ -31,39 +59,149 @@ export default function App(): React.ReactElement | null {
       });
     }
   }, [camPerm, micPerm]);
-  console.log("App", permsGranted);
+
+  const [objectFrames, setObjectFrames] = React.useState<Detection[]>([]);
+
+  const frameProcessor = useObjectDetection(
+    (results) => {
+      console.log(results);
+      const firstResult = results.results[0];
+      const detections = firstResult?.detections ?? [];
+      setObjectFrames(
+        detections.map((detection) => {
+          return {
+            label: detection.categories[0]?.categoryName ?? "unknown",
+            x: (detection.boundingBox.left / results.inputImageWidth) * width,
+            y: (detection.boundingBox.top / results.inputImageHeight) * height,
+            width:
+              ((detection.boundingBox.right - detection.boundingBox.left) /
+                results.inputImageWidth) *
+              width,
+            height:
+              ((detection.boundingBox.bottom - detection.boundingBox.top) /
+                results.inputImageHeight) *
+              height,
+          };
+        })
+      );
+    },
+    (error) => {
+      console.error(`onError: ${error}`);
+    },
+    RunningMode.LIVE_STREAM,
+    "efficientdet-lite0.tflite",
+    { delegate: Delegate.GPU }
+  );
+  if (permsGranted.cam && permsGranted.mic) {
+    return (
+      <View style={styles.container}>
+        <MediapipeCamera style={styles.box} processor={frameProcessor} />
+        <Canvas style={styles.box}>
+          {objectFrames.map((frame, index) => (
+            <ObjectFrame frame={frame} index={index} key={index} />
+          ))}
+        </Canvas>
+      </View>
+    );
+  } else {
+    return <NeedPermissions askForPermissions={askForPermissions} />;
+  }
+}
+
+const NeedPermissions: React.FC<{ askForPermissions: () => void }> = ({
+  askForPermissions,
+}) => {
   return (
     <View style={styles.container}>
-      {permsGranted.cam && permsGranted.mic ? (
-        <MediapipeCamera style={styles.box} />
-      ) : (
-        <>
-          <Text style={styles.noPermsText}>
-            Camera and Mic permissions required
-          </Text>
-          <Pressable style={styles.permsButton} onPress={askForPermissions}>
-            <Text>Request</Text>
-          </Pressable>
-        </>
-      )}
+      <Text style={styles.noPermsText}>
+        Camera and Mic permissions required
+      </Text>
+      <Pressable style={styles.permsButton} onPress={askForPermissions}>
+        <Text>Request</Text>
+      </Pressable>
     </View>
   );
-}
+};
+
+const ObjectFrame: React.FC<{ frame: Detection; index: number }> = ({
+  frame,
+  index,
+}) => {
+  const color = colorNames[index % colorNames.length];
+  return (
+    <Group style={"stroke"} strokeWidth={2}>
+      <SkiaText
+        x={frame.x}
+        y={frame.y}
+        color={color}
+        text={frame.label}
+        font={font}
+      />
+      <Rect
+        key={index}
+        x={frame.x}
+        y={frame.y}
+        width={frame.width}
+        height={frame.height}
+        color={color}
+      />
+    </Group>
+  );
+};
 
 const styles = StyleSheet.create({
   container: {
     flex: 1,
     alignItems: "center",
     justifyContent: "center",
+    position: "relative",
   },
   box: {
-    flex: 1,
-    alignSelf: "stretch",
+    position: "absolute",
+    top: 0,
+    left: 0,
+    width: "100%",
+    height: "100%",
+  },
+  permsButton: {
+    padding: 10,
+    backgroundColor: "lightblue",
+    borderRadius: 5,
+    margin: 10,
   },
-  permsButton: {},
   noPermsText: {
     fontSize: 20,
     fontWeight: "bold",
     color: "red",
   },
 });
+
+const colorNames = [
+  "Coral",
+  "DarkCyan",
+  "DeepSkyBlue",
+  "ForestGreen",
+  "GoldenRod",
+  "MediumOrchid",
+  "SteelBlue",
+  "Tomato",
+  "Turquoise",
+  "SlateGray",
+  "DodgerBlue",
+  "FireBrick",
+  "Gold",
+  "HotPink",
+  "LimeGreen",
+  "Navy",
+  "OrangeRed",
+  "RoyalBlue",
+  "SeaGreen",
+  "Violet",
+];
+
+const fontFamily = Platform.select({ ios: "Helvetica", android: "sans-serif" });
+const fontStyle = {
+  fontFamily,
+  fontSize: 14,
+};
+const font = matchFont(fontStyle);

ios/objectdetection/ObjectDetectionFrameProcessorPlugin.swift

@@ -17,11 +17,15 @@ public class ObjectDetectionFrameProcessorPlugin: FrameProcessorPlugin {
 
   public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any
   {
-    let detectorHandle: Double = arguments?["detectorHandle"] as! Double
-    guard let detector = ObjectDetectionModule.detectorMap[Int(detectorHandle)] else {
+    guard let detectorHandleValue = arguments?["detectorHandle"] as? Double else {
       return false
     }
-
+
+    // Now that we have a valid Double, attempt to retrieve the detector using it
+    guard let detector = ObjectDetectionModule.detectorMap[Int(detectorHandleValue)] else {
+      return false
+    }
+
     let buffer = frame.buffer
     detector.detectAsync(
       sampleBuffer: buffer,