From ccdf2b3c804a7bcd92a6fd8ca8ebdebe5811e68f Mon Sep 17 00:00:00 2001 From: Charles Parker Date: Sun, 28 Apr 2024 16:31:11 -0700 Subject: [PATCH] fix: more get it working --- examples/objectdetection/ios/Podfile.lock | 2 +- .../objectdetection.xcodeproj/project.pbxproj | 12 +- examples/objectdetection/src/CameraStream.tsx | 44 +++--- src/index.tsx | 36 +---- src/objectDetection/index.ts | 51 +++++-- src/shared/convert.ts | 128 ++++++++++++++++++ src/shared/mediapipeCamera.tsx | 42 ++++++ src/shared/types.ts | 8 ++ tsconfig.shared.json | 1 + 9 files changed, 253 insertions(+), 71 deletions(-) create mode 100644 src/shared/convert.ts create mode 100644 src/shared/mediapipeCamera.tsx create mode 100644 src/shared/types.ts diff --git a/examples/objectdetection/ios/Podfile.lock b/examples/objectdetection/ios/Podfile.lock index 183ca0e..80bc421 100644 --- a/examples/objectdetection/ios/Podfile.lock +++ b/examples/objectdetection/ios/Podfile.lock @@ -1435,4 +1435,4 @@ SPEC CHECKSUMS: PODFILE CHECKSUM: 106c37d775a4ea3a9fa9744362f0af5ba16aac0e -COCOAPODS: 1.15.2 +COCOAPODS: 1.14.3 diff --git a/examples/objectdetection/ios/objectdetection.xcodeproj/project.pbxproj b/examples/objectdetection/ios/objectdetection.xcodeproj/project.pbxproj index d844346..749986f 100644 --- a/examples/objectdetection/ios/objectdetection.xcodeproj/project.pbxproj +++ b/examples/objectdetection/ios/objectdetection.xcodeproj/project.pbxproj @@ -586,7 +586,11 @@ "-DFOLLY_USE_LIBCPP=1", "-DFOLLY_CFG_NO_COROUTINES=1", ); - OTHER_LDFLAGS = "$(inherited)"; + OTHER_LDFLAGS = ( + "$(inherited)", + "-Wl", + "-ld_classic", + ); REACT_NATIVE_PATH = "${PODS_ROOT}/../../node_modules/react-native"; SDKROOT = iphoneos; USE_HERMES = true; @@ -654,7 +658,11 @@ "-DFOLLY_USE_LIBCPP=1", "-DFOLLY_CFG_NO_COROUTINES=1", ); - OTHER_LDFLAGS = "$(inherited)"; + OTHER_LDFLAGS = ( + "$(inherited)", + "-Wl", + "-ld_classic", + ); REACT_NATIVE_PATH = "${PODS_ROOT}/../../node_modules/react-native"; SDKROOT = iphoneos; USE_HERMES = true; diff --git a/examples/objectdetection/src/CameraStream.tsx b/examples/objectdetection/src/CameraStream.tsx index 24dfa10..f97daf5 100644 --- a/examples/objectdetection/src/CameraStream.tsx +++ b/examples/objectdetection/src/CameraStream.tsx @@ -7,14 +7,7 @@ import { } from "@shopify/react-native-skia"; import * as React from "react"; -import { - Platform, - Pressable, - StyleSheet, - Text, - View, - useWindowDimensions, -} from "react-native"; +import { Platform, Pressable, StyleSheet, Text, View } from "react-native"; import { Delegate, MediapipeCamera, @@ -29,6 +22,7 @@ import { } from "react-native-vision-camera"; import type { RootTabParamList } from "./navigation"; import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs"; +import { frameRectToView, ltrbToXywh } from "../../../src/shared/convert"; interface Detection { label: string; @@ -41,7 +35,6 @@ interface Detection { type Props = BottomTabScreenProps; export const CameraStream: React.FC = () => { - const { width, height } = useWindowDimensions(); const camPerm = useCameraPermission(); const micPerm = useMicrophonePermission(); const [permsGranted, setPermsGranted] = React.useState<{ @@ -75,25 +68,28 @@ export const CameraStream: React.FC = () => { ); }; - const frameProcessor = useObjectDetection( - (results) => { - console.log(results); + const objectDetection = useObjectDetection( + (results, viewSize) => { const firstResult = results.results[0]; const detections = firstResult?.detections ?? []; + const frameSize = { + width: results.inputImageWidth, + height: results.inputImageHeight, + }; setObjectFrames( detections.map((detection) => { + const { x, y, width, height } = frameRectToView( + ltrbToXywh(detection.boundingBox), + frameSize, + viewSize, + "cover" + ); return { label: detection.categories[0]?.categoryName ?? "unknown", - x: (detection.boundingBox.left / results.inputImageWidth) * width, - y: (detection.boundingBox.top / results.inputImageHeight) * height, - width: - ((detection.boundingBox.right - detection.boundingBox.left) / - results.inputImageWidth) * - width, - height: - ((detection.boundingBox.bottom - detection.boundingBox.top) / - results.inputImageHeight) * - height, + x, + y, + width, + height, }; }) ); @@ -111,8 +107,9 @@ export const CameraStream: React.FC = () => { {objectFrames.map((frame, index) => ( @@ -172,6 +169,7 @@ const ObjectFrame: React.FC<{ frame: Detection; index: number }> = ({ const styles = StyleSheet.create({ container: { + backgroundColor: "red", flex: 1, alignItems: "center", justifyContent: "center", diff --git a/src/index.tsx b/src/index.tsx index 1d6e149..202c563 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,35 +1,3 @@ -import React from "react"; -import { type ViewStyle, Text, Platform } from "react-native"; -import { - Camera, - useCameraDevice, - type CameraPosition, - type FrameProcessor, -} from "react-native-vision-camera"; - -export type MediapipeCameraProps = { - style: ViewStyle; - processor: FrameProcessor; - activeCamera: CameraPosition; -}; - -export const MediapipeCamera: React.FC = ({ - style, - processor, - activeCamera, -}) => { - const device = useCameraDevice(activeCamera); - return device !== undefined ? ( - - ) : ( - no device - ); -}; - export * from "./objectDetection"; +export * from "./shared/mediapipeCamera"; +export * from "./shared/convert"; diff --git a/src/objectDetection/index.ts b/src/objectDetection/index.ts index 0516a7e..a1b3430 100644 --- a/src/objectDetection/index.ts +++ b/src/objectDetection/index.ts @@ -1,9 +1,15 @@ import React from "react"; -import { NativeEventEmitter, NativeModules } from "react-native"; +import { + NativeEventEmitter, + NativeModules, + type LayoutChangeEvent, +} from "react-native"; import { VisionCameraProxy, useFrameProcessor, } from "react-native-vision-camera"; +import type { MediaPipeSolution } from "../shared/types"; +import type { Dims } from "../shared/convert"; const { ObjectDetection } = NativeModules; const eventEmitter = new NativeEventEmitter(ObjectDetection); @@ -96,8 +102,9 @@ export interface ObjectDetectionOptions { resize: { scale: number; aspect: "preserve" | "default" | number }; } export interface ObjectDetectionCallbacks { - onResults: (result: ResultBundleMap) => void; + onResults: (result: ResultBundleMap, viewSize: Dims) => void; onError: (error: ObjectDetectionError) => void; + viewSize: Dims; } // TODO setup the general event callbacks @@ -107,7 +114,7 @@ eventEmitter.addListener( (args: { handle: number } & ResultBundleMap) => { const callbacks = detectorMap.get(args.handle); if (callbacks) { - callbacks.onResults(args); + callbacks.onResults(args, callbacks.viewSize); } } ); @@ -127,17 +134,36 @@ export function useObjectDetection( runningMode: RunningMode, model: string, options?: Partial -) { +): MediaPipeSolution { const [detectorHandle, setDetectorHandle] = React.useState< number | undefined >(); + const [cameraViewDimensions, setCameraViewDimensions] = React.useState<{ + width: number; + height: number; + }>({ width: 1, height: 1 }); + + const cameraViewLayoutChangeHandler = React.useCallback( + (event: LayoutChangeEvent) => { + setCameraViewDimensions({ + height: event.nativeEvent.layout.height, + width: event.nativeEvent.layout.width, + }); + }, + [] + ); + // Remember the latest callback if it changes. React.useLayoutEffect(() => { if (detectorHandle !== undefined) { - detectorMap.set(detectorHandle, { onResults, onError }); + detectorMap.set(detectorHandle, { + onResults, + onError, + viewSize: cameraViewDimensions, + }); } - }, [onResults, onError, detectorHandle]); + }, [onResults, onError, detectorHandle, cameraViewDimensions]); React.useEffect(() => { let newHandle: number | undefined; @@ -182,15 +208,18 @@ export function useObjectDetection( plugin?.call(frame, { detectorHandle, - scale: { - width: frame.width * 0.5, - height: frame.height * 0.5, - }, pixelFormat: "rgb", dataType: "uint8", }); }, [detectorHandle] ); - return frameProcessor; + return React.useMemo( + (): MediaPipeSolution => ({ + cameraViewLayoutChangeHandler, + cameraViewDimensions, + frameProcessor, + }), + [cameraViewDimensions, cameraViewLayoutChangeHandler, frameProcessor] + ); } diff --git a/src/shared/convert.ts b/src/shared/convert.ts new file mode 100644 index 0000000..be6719a --- /dev/null +++ b/src/shared/convert.ts @@ -0,0 +1,128 @@ +export type Dims = { width: number; height: number }; +export type Point = { x: number; y: number }; +export type RectXYWH = { x: number; y: number; width: number; height: number }; +export type RectLTRB = { + left: number; + top: number; + right: number; + bottom: number; +}; +export type ResizeMode = "cover" | "contain"; + +// both cover and contain preserve aspect ratio. Cover will crop the image to fill the view, contain will show the whole image and add padding. +// for cover, if the aspect ratio x/y of the frame is greater than +export function framePointToView( + point: Point, + frameDims: Dims, + viewDims: Dims, + mode: ResizeMode +): Point { + const frameRatio = frameDims.width / frameDims.height; + const viewRatio = viewDims.width / viewDims.height; + let scale = 1; + let xoffset = 0; + let yoffset = 0; + if (mode === "contain") { + // contain means that the frame rect will be smaller than the view rect, + // if the w/h ratio of the frame is greater than the w/h ratio of the view, + // then equal in the x dimension, smaller in the y dimension + // else the other way around + if (frameRatio > viewRatio) { + scale = viewDims.width / frameDims.width; + xoffset = 0; + yoffset = (viewDims.height - frameDims.height * scale) / 2; + } else { + scale = viewDims.height / frameDims.height; + xoffset = (viewDims.width - frameDims.width * scale) / 2; + yoffset = 0; + } + } else { + if (frameRatio > viewRatio) { + scale = viewDims.height / frameDims.height; + xoffset = (viewDims.width - frameDims.width * scale) / 2; + yoffset = 0; + } else { + scale = viewDims.width / frameDims.width; + xoffset = 0; + yoffset = (viewDims.height - frameDims.height * scale) / 2; + } + } + return { + x: point.x * scale + xoffset, + y: point.y * scale + yoffset, + }; +} + +function frameRectLTRBToView( + rect: RectLTRB, + frameDims: Dims, + viewDims: Dims, + mode: ResizeMode +): RectLTRB { + const lt = framePointToView( + { x: rect.left, y: rect.top }, + frameDims, + viewDims, + mode + ); + const rb = framePointToView( + { x: rect.right, y: rect.bottom }, + frameDims, + viewDims, + mode + ); + return { left: lt.x, top: lt.y, right: rb.x, bottom: rb.y }; +} + +function frameRectXYWHToView( + rect: RectXYWH, + frameDims: Dims, + viewDims: Dims, + mode: ResizeMode +): RectXYWH { + const lt = framePointToView( + { x: rect.x, y: rect.y }, + frameDims, + viewDims, + mode + ); + const rb = framePointToView( + { x: rect.x + rect.width, y: rect.y + rect.height }, + frameDims, + viewDims, + mode + ); + return { x: lt.x, y: lt.y, width: rb.x - lt.x, height: rb.y - lt.y }; +} + +function isRectLTRB(rect: unknown): rect is RectLTRB { + return ( + typeof rect === "object" && + "left" in (rect as object) && + "top" in (rect as object) && + "right" in (rect as object) && + "bottom" in (rect as object) + ); +} + +export function frameRectToView( + rect: TRect, + frameDims: Dims, + viewDims: Dims, + mode: ResizeMode +): TRect { + if (isRectLTRB(rect)) { + return frameRectLTRBToView(rect, frameDims, viewDims, mode) as TRect; + } else { + return frameRectXYWHToView(rect, frameDims, viewDims, mode) as TRect; + } +} + +export function ltrbToXywh(rect: RectLTRB): RectXYWH { + return { + x: rect.left, + y: rect.top, + width: rect.right - rect.left, + height: rect.bottom - rect.top, + }; +} diff --git a/src/shared/mediapipeCamera.tsx b/src/shared/mediapipeCamera.tsx new file mode 100644 index 0000000..de92d6d --- /dev/null +++ b/src/shared/mediapipeCamera.tsx @@ -0,0 +1,42 @@ +import React from "react"; +import { type ViewStyle, Text, Platform } from "react-native"; +import { + Camera, + useCameraDevice, + type CameraPosition, + type CameraProps, + type Orientation, +} from "react-native-vision-camera"; +import type { MediaPipeSolution } from "./types"; + +export type MediapipeCameraProps = { + style: ViewStyle; + solution: MediaPipeSolution; + activeCamera?: CameraPosition; + orientation?: Orientation; + resizeMode?: CameraProps["resizeMode"]; +}; + +export const MediapipeCamera: React.FC = ({ + style, + solution, + activeCamera = "front", + orientation = "portrait", + resizeMode = "cover", +}) => { + const device = useCameraDevice(activeCamera); + return device !== undefined ? ( + + ) : ( + no device + ); +}; diff --git a/src/shared/types.ts b/src/shared/types.ts new file mode 100644 index 0000000..35bbe62 --- /dev/null +++ b/src/shared/types.ts @@ -0,0 +1,8 @@ +import type { LayoutChangeEvent } from "react-native"; +import type { FrameProcessor } from "react-native-vision-camera"; + +export interface MediaPipeSolution { + frameProcessor: FrameProcessor; + cameraViewLayoutChangeHandler: (event: LayoutChangeEvent) => void; + cameraViewDimensions: { width: number; height: number }; +} diff --git a/tsconfig.shared.json b/tsconfig.shared.json index cb22707..7b69b82 100644 --- a/tsconfig.shared.json +++ b/tsconfig.shared.json @@ -10,6 +10,7 @@ ], "module": "esnext", "moduleResolution": "node", + "noEmit": true, "noFallthroughCasesInSwitch": true, "noImplicitReturns": true, "noImplicitUseStrict": false,