Skip to content

Commit

Permalink
feat(settings): add a simple settings menu to the objectdetection sam…
Browse files Browse the repository at this point in the history
…ple (#79)

* feat(settings): add slider ui for max results and score threshold

* feat(settings): add in pickers for delegate and model selections

* feat(settings): create an applications state and modify settings there

* feat(settings): get ios version working

* fix(settings): get picker working on Android

* fix(settings): make the camera stream respect settings

* fix(settings): use debounce to prevent reloading the object detector too frequently

* fix(settings): get ios working with settings

This reverts commit 075206b.

* fix(settings): comment out model chooser temporarily

* fix(image-picker): update podfile
  • Loading branch information
dwgray authored May 9, 2024
1 parent a4f7d03 commit 0b6bc95
Show file tree
Hide file tree
Showing 9 changed files with 2,335 additions and 903 deletions.
32 changes: 31 additions & 1 deletion examples/objectdetection/ios/Podfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -955,6 +955,10 @@ PODS:
- React
- React-callinvoker
- React-Core
- react-native-slider (4.5.2):
- glog
- RCT-Folly (= 2022.05.16.00)
- React-Core
- react-native-worklets-core (1.3.0):
- React
- React-callinvoker
Expand Down Expand Up @@ -1132,6 +1136,17 @@ PODS:
- RCT-Folly (= 2022.05.16.00)
- React-Core
- VisionCamera
- RNCPicker (2.7.5):
- React-Core
- RNImageCropPicker (0.40.3):
- React-Core
- React-RCTImage
- RNImageCropPicker/QBImagePickerController (= 0.40.3)
- TOCropViewController
- RNImageCropPicker/QBImagePickerController (0.40.3):
- React-Core
- React-RCTImage
- TOCropViewController
- RNScreens (3.30.1):
- glog
- RCT-Folly (= 2022.05.16.00)
Expand All @@ -1141,6 +1156,7 @@ PODS:
- RCT-Folly (= 2022.05.16.00)
- React-Core
- SocketRocket (0.6.1)
- TOCropViewController (2.7.4)
- VisionCamera (4.0.1):
- VisionCamera/Core (= 4.0.1)
- VisionCamera/FrameProcessors (= 4.0.1)
Expand Down Expand Up @@ -1210,6 +1226,7 @@ DEPENDENCIES:
- React-Mapbuffer (from `../node_modules/react-native/ReactCommon`)
- react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
- "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
- "react-native-slider (from `../node_modules/@react-native-community/slider`)"
- react-native-worklets-core (from `../node_modules/react-native-worklets-core`)
- React-nativeconfig (from `../node_modules/react-native/ReactCommon`)
- React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
Expand All @@ -1232,6 +1249,8 @@ DEPENDENCIES:
- React-utils (from `../node_modules/react-native/ReactCommon/react/utils`)
- ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
- ReactNativeMediaPipe (from `../../..`)
- "RNCPicker (from `../node_modules/@react-native-picker/picker`)"
- RNImageCropPicker (from `../node_modules/react-native-image-crop-picker`)
- RNScreens (from `../node_modules/react-native-screens`)
- RNVectorIcons (from `../node_modules/react-native-vector-icons`)
- VisionCamera (from `../node_modules/react-native-vision-camera`)
Expand All @@ -1254,6 +1273,7 @@ SPEC REPOS:
- MediaPipeTasksVision
- OpenSSL-Universal
- SocketRocket
- TOCropViewController

EXTERNAL SOURCES:
boost:
Expand Down Expand Up @@ -1315,6 +1335,8 @@ EXTERNAL SOURCES:
:path: "../node_modules/react-native-safe-area-context"
react-native-skia:
:path: "../node_modules/@shopify/react-native-skia"
react-native-slider:
:path: "../node_modules/@react-native-community/slider"
react-native-worklets-core:
:path: "../node_modules/react-native-worklets-core"
React-nativeconfig:
Expand Down Expand Up @@ -1359,6 +1381,10 @@ EXTERNAL SOURCES:
:path: "../node_modules/react-native/ReactCommon"
ReactNativeMediaPipe:
:path: "../../.."
RNCPicker:
:path: "../node_modules/@react-native-picker/picker"
RNImageCropPicker:
:path: "../node_modules/react-native-image-crop-picker"
RNScreens:
:path: "../node_modules/react-native-screens"
RNVectorIcons:
Expand Down Expand Up @@ -1412,6 +1438,7 @@ SPEC CHECKSUMS:
React-Mapbuffer: 84ea43c6c6232049135b1550b8c60b2faac19fab
react-native-safe-area-context: b97eb6f9e3b7f437806c2ce5983f479f8eb5de4b
react-native-skia: 6da30aeaa315fcec5c27ebfb016b9eca2b57b66f
react-native-slider: 7a39874fc1fcdfee48e448fa72cce0a8f2c7c5d6
react-native-worklets-core: 7a50c0c14005cc57df8583eabd99ea3f467fa277
React-nativeconfig: b4d4e9901d4cabb57be63053fd2aa6086eb3c85f
React-NativeModulesApple: cd26e56d56350e123da0c1e3e4c76cb58a05e1ee
Expand All @@ -1434,12 +1461,15 @@ SPEC CHECKSUMS:
React-utils: d16c1d2251c088ad817996621947d0ac8167b46c
ReactCommon: 2aa35648354bd4c4665b9a5084a7d37097b89c10
ReactNativeMediaPipe: 92850eb7623b4bb0d6e8fe90c5e5781b42ccba00
RNCPicker: 3e2c37a8328f368ce14da050cdc8231deb5fc9f9
RNImageCropPicker: e7ab6fb43d2fc3e84651e786ef4a080d63b0ed3d
RNScreens: b6b64d956af3715adbfe84808694ae82d3fec74f
RNVectorIcons: 73ab573085f65a572d3b6233e68996d4707fd505
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
TOCropViewController: 80b8985ad794298fb69d3341de183f33d1853654
VisionCamera: 8e00df84a76cf26ca70ecd3b6ed05dc0d3b60beb
Yoga: 805bf71192903b20fc14babe48080582fee65a80

PODFILE CHECKSUM: 106c37d775a4ea3a9fa9744362f0af5ba16aac0e

COCOAPODS: 1.14.3
COCOAPODS: 1.15.2
3 changes: 3 additions & 0 deletions examples/objectdetection/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,16 @@
"test": "jest"
},
"dependencies": {
"@react-native-community/slider": "^4.5.2",
"@react-native-picker/picker": "^2.7.5",
"@react-navigation/bottom-tabs": "^6.5.20",
"@react-navigation/native": "^6.1.17",
"@shopify/react-native-skia": "^1.0.5",
"@types/react-native-vector-icons": "^6.4.18",
"react": "18.2.0",
"react-native": "0.73.6",
"react-native-image-crop-picker": "^0.40.2",
"react-native-picker-select": "^9.1.3",
"react-native-safe-area-context": "^4.9.0",
"react-native-screens": "^3.30.1",
"react-native-vector-icons": "^10.0.3",
Expand Down
68 changes: 40 additions & 28 deletions examples/objectdetection/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import { NavigationContainer, type RouteProp } from "@react-navigation/native";
import { createBottomTabNavigator } from "@react-navigation/bottom-tabs";
import { type RootTabParamList } from "./navigation";
import Ionicons from "react-native-vector-icons/Ionicons";
import type { AppSettings } from "./app-settings";
import { Delegate } from "react-native-mediapipe";
import { SettingsContext } from "./app-settings";

const Tab = createBottomTabNavigator<RootTabParamList>();

Expand Down Expand Up @@ -37,35 +40,44 @@ const RenderTabBarIcon: React.FC<TabBarIconProps> = ({
};

function App() {
const [settings, setSettings] = React.useState<AppSettings>({
maxResults: 5,
threshold: 20,
processor: Delegate.GPU,
model: "efficientdet-lite0",
});

return (
<NavigationContainer>
<Tab.Navigator
initialRouteName="CameraStream"
screenOptions={({ route }) => ({
tabBarIcon: ({ focused, color, size }) => {
return RenderTabBarIcon({ focused, color, size, route });
},
tabBarActiveTintColor: "tomato",
tabBarInactiveTintColor: "gray",
})}
>
<Tab.Screen
name="CameraStream"
component={CameraStream}
options={{ title: "Camera" }}
/>
<Tab.Screen
name="Photo"
component={Photo}
options={{ title: "Photos" }}
/>
<Tab.Screen
name="Settings"
component={Settings}
options={{ title: "Settings" }}
/>
</Tab.Navigator>
</NavigationContainer>
<SettingsContext.Provider value={{ settings, setSettings }}>
<NavigationContainer>
<Tab.Navigator
initialRouteName="CameraStream"
screenOptions={({ route }) => ({
tabBarIcon: ({ focused, color, size }) => {
return RenderTabBarIcon({ focused, color, size, route });
},
tabBarActiveTintColor: "tomato",
tabBarInactiveTintColor: "gray",
})}
>
<Tab.Screen
name="CameraStream"
component={CameraStream}
options={{ title: "Camera" }}
/>
<Tab.Screen
name="Photo"
component={Photo}
options={{ title: "Photos" }}
/>
<Tab.Screen
name="Settings"
component={Settings}
options={{ title: "Settings" }}
/>
</Tab.Navigator>
</NavigationContainer>
</SettingsContext.Provider>
);
}

Expand Down
13 changes: 10 additions & 3 deletions examples/objectdetection/src/CameraStream.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import * as React from "react";

import { Platform, Pressable, StyleSheet, Text, View } from "react-native";
import {
Delegate,
MediapipeCamera,
RunningMode,
useObjectDetection,
Expand All @@ -25,6 +24,8 @@ import {
import type { RootTabParamList } from "./navigation";
import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs";
import { frameRectToView, ltrbToXywh } from "../../../src/shared/convert";
import { useSettings } from "./app-settings";
import { useDebounce } from "./useDebounce";

interface Detection {
label: string;
Expand All @@ -37,6 +38,8 @@ interface Detection {
type Props = BottomTabScreenProps<RootTabParamList, "CameraStream">;

export const CameraStream: React.FC<Props> = () => {
const { settings } = useSettings();
const debouncedSettings = useDebounce(settings, 500);
const camPerm = useCameraPermission();
const micPerm = useMicrophonePermission();
const [permsGranted, setPermsGranted] = React.useState<{
Expand Down Expand Up @@ -100,8 +103,12 @@ export const CameraStream: React.FC<Props> = () => {
console.error(`onError: ${error}`);
},
RunningMode.LIVE_STREAM,
"efficientdet-lite0.tflite",
{ delegate: Delegate.GPU }
`${debouncedSettings.model}.tflite`,
{
delegate: debouncedSettings.processor,
maxResults: debouncedSettings.maxResults,
threshold: debouncedSettings.threshold / 100,
}
);

if (permsGranted.cam && permsGranted.mic) {
Expand Down
119 changes: 116 additions & 3 deletions examples/objectdetection/src/Settings.tsx
Original file line number Diff line number Diff line change
@@ -1,14 +1,127 @@
import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs";
import React from "react";
import { View, Text } from "react-native";
import { View, Text, StyleSheet } from "react-native";
import type { RootTabParamList } from "./navigation";
import Slider from "@react-native-community/slider";
import RNPickerSelect from "react-native-picker-select";
import { Delegate } from "react-native-mediapipe";
import { useSettings } from "./app-settings";

type SlidersComponentProps = {
label: string;
value: number;
setValue: (value: number) => void;
minValue?: number;
maxValue?: number;
};

type SelectComponentProps = {
label: string;
value: unknown;
setValue: (value: unknown) => void;
items: { label: string; value: unknown }[];
};

const OptionSlider: React.FC<SlidersComponentProps> = ({
label,
value,
setValue,
minValue = 1,
maxValue = 10,
}) => {
return (
<View style={styles.item}>
<Text style={styles.label}>
{label.replace("${value}", value.toString())}
</Text>
<Slider
value={value}
onValueChange={setValue}
minimumValue={minValue}
maximumValue={maxValue}
step={1}
style={styles.slider}
/>
</View>
);
};

const OptionSelect: React.FC<SelectComponentProps> = ({
label,
value,
setValue,
items,
}) => {
return (
<View style={styles.item}>
<Text style={styles.label}>{label}</Text>
<RNPickerSelect
value={value}
onValueChange={setValue}
style={{ inputAndroid: styles.picker, inputIOS: styles.picker }}
useNativeAndroidPickerStyle={false}
items={items}
/>
</View>
);
};

type Props = BottomTabScreenProps<RootTabParamList, "Settings">;

export const Settings: React.FC<Props> = () => {
const { settings, setSettings } = useSettings();

return (
<View style={{ flex: 1, alignItems: "center", justifyContent: "center" }}>
<Text>Settings Placeholder</Text>
<View style={styles.container}>
<OptionSelect
label="Processor: "
value={settings.processor}
setValue={(value) =>
setSettings({ ...settings, processor: value as Delegate })
}
items={[
{ label: "GPU", value: Delegate.GPU },
{ label: "CPU", value: Delegate.CPU },
]}
/>
{/* Disable Model selection temporarily
<OptionSelect
label="Model selections: "
value={settings.model}
setValue={(value) =>
setSettings({ ...settings, model: value as string })
}
items={[
{ label: "EfficientDet-Lite0", value: "efficientdet-lite0" },
{ label: "EfficientDet-Lite2", value: "efficientdet-lite2" },
{ label: "SSD MobileNetV2", value: "ssd-mobilenetv2" },
]}
/> */}
<OptionSlider
label="Max results: ${value}"
value={settings.maxResults}
setValue={(value) => setSettings({ ...settings, maxResults: value })}
/>
<OptionSlider
label="Score threshold: ${value}%"
value={settings.threshold}
setValue={(value) => setSettings({ ...settings, threshold: value })}
minValue={0}
maxValue={100}
/>
</View>
);
};

const styles = StyleSheet.create({
container: {
flex: 1,
alignItems: "flex-start",
justifyContent: "flex-start",
position: "relative",
},
item: { padding: 10 },
slider: { width: 250, height: 40 },
picker: { width: 250, height: 40, marginLeft: 15 },
label: { marginLeft: 15 },
});
Loading

0 comments on commit 0b6bc95

Please sign in to comment.