Version 2.0.1 seems to fail on Expo 50 and 1.0.0 seems to hang

[asnaseer]

Background
We have an Expo 50 React Native app written in Typescript that makes use of our backend API for object detection in images captured by our users. We wanted to try out your library to see if we can do this on-device (iOS and Android devices).

Our app uses these relevant libs in package.json:

    "@expo-google-fonts/rubik": "^0.1.0",
    "@expo/metro-config": "~0.17.1",
    "@expo/react-native-action-sheet": "^3.9.0",
    "@expo/vector-icons": "^14.0.0",
    "@react-native-async-storage/async-storage": "1.21.0",
    "@react-native-community/blur": "^4.4.0",
    "@react-native-community/datetimepicker": "^7.6.3",
    "@react-native-community/masked-view": "0.1.10",
    "@react-native-community/netinfo": "11.1.0",
    "@react-native-firebase/analytics": "^20.3.0",
    "@react-native-firebase/app": "^20.3.0",
    "@react-native-google-signin/google-signin": "^10.0.1",
    "@react-native-voice/voice": "^3.2.4",
    "@react-navigation/bottom-tabs": "^6.5.20",
    "@react-navigation/drawer": "^6.6.15",
    "@react-navigation/material-top-tabs": "^6.6.13",
    "@react-navigation/native": "^6.1.17",
    "@react-navigation/stack": "^6.3.29",
    "expo": "~50.0.17",
    "expo-apple-authentication": "~6.3.0",
    "expo-application": "~5.8.4",
    "expo-asset": "~9.0.2",
    "expo-av": "~13.10.5",
    "expo-build-properties": "~0.11.1",
    "expo-camera": "~14.1.3",
    "expo-clipboard": "~5.0.1",
    "expo-constants": "~15.4.6",
    "expo-contacts": "~12.8.2",
    "expo-crypto": "~12.8.1",
    "expo-dev-client": "~3.3.11",
    "expo-device": "~5.9.4",
    "expo-file-system": "~16.0.9",
    "expo-font": "~11.10.3",
    "expo-gl": "~13.6.0",
    "expo-haptics": "~12.8.1",
    "expo-image-picker": "~14.7.1",
    "expo-intent-launcher": "~10.11.0",
    "expo-linear-gradient": "~12.7.2",
    "expo-linking": "~6.2.2",
    "expo-localization": "~14.8.4",
    "expo-location": "~16.5.5",
    "expo-mail-composer": "~12.7.1",
    "expo-media-library": "~15.9.2",
    "expo-notifications": "~0.27.7",
    "expo-random": "~13.6.0",
    "expo-sms": "~11.7.1",
    "expo-splash-screen": "~0.26.4",
    "expo-sqlite": "~13.4.0",
    "expo-status-bar": "~1.11.1",
    "expo-store-review": "~6.8.3",
    "expo-task-manager": "~11.7.3",
    "expo-tracking-transparency": "~3.3.0",
    "expo-updates": "~0.24.12",
    "expo-web-browser": "~12.8.2",
    "react": "18.2.0",
    "react-content-loader": "^6.2.0",
    "react-dom": "18.2.0",
    "react-hook-form": "^6.15.1",
    "react-native": "0.73.6",
    "react-native-animatable": "^1.3.3",
    "react-native-animated-spinkit": "^1.5.1",
    "react-native-appstate-hook": "^1.0.6",
    "react-native-branch": "^6.2.1",
    "react-native-calendars": "1.1294.0",
    "react-native-canvas": "^0.1.40",
    "react-native-circular-progress": "^1.3.7",
    "react-native-collapsible": "^1.6.0",
    "react-native-confetti-cannon": "^1.5.1",
    "react-native-drag-sort": "^2.4.4",
    "react-native-fast-image": "^8.5.11",
    "react-native-fbsdk-next": "^11.2.1",
    "react-native-fs": "^2.20.0",
    "react-native-gesture-handler": "~2.14.0",
    "react-native-get-random-values": "~1.8.0",
    "react-native-image-colors": "^1.5.2",
    "react-native-image-crop-picker": "^0.41.2",
    "react-native-image-viewing": "git+https://github.com/ChirpBirding/react-native-image-viewing.git#906090627dcd029164ed26fc6108abe640bf2e75",
    "react-native-live-audio-stream": "^1.1.1",
    "react-native-localize": "^3.0.6",
    "react-native-markdown-display": "^7.0.0-alpha.2",
    "react-native-mime-types": "^2.3.0",
    "react-native-pager-view": "6.1.4",
    "react-native-purchases": "^7.5.0",
    "react-native-reanimated": "~3.6.2",
    "react-native-safe-area-context": "4.8.2",
    "react-native-screens": "~3.29.0",
    "react-native-share": "^8.2.2",
    "react-native-sse": "^1.2.1",
    "react-native-status-bar-height": "^2.6.0",
    "react-native-svg": "14.1.0",
    "react-native-svg-charts": "^5.4.0",
    "react-native-tab-view": "^3.5.2",
    "react-native-url-polyfill": "^1.3.0",
    "react-native-view-shot": "3.8.0",
    "react-native-vision-camera": "4.5.1",
    "react-native-web": "~0.19.6",
    "react-native-webview": "^13.10.5",
    "react-native-zip-archive": "^6.0.5",
    "react-use": "^17.2.4",
    "typescript": "^4.9.5",

Issue
We installed the latest version of your library (v2.0.1) and then tried a simple thing where we just added your provider into our App.tsx file as follows:

...
import { useObjectDetectionModels } from "@infinitered/react-native-mlkit-object-detection";
...
export default function App(): JSX.Element | null {
  ...
  const { ObjectDetectionModelContextProvider } = useObjectDetectionModels({
    loadDefaultModel: true,
    defaultModelOptions: {
      shouldEnableMultipleObjects: true,
      shouldEnableClassification: true,
      detectorMode: "singleImage",
    },
  });
...
  return (
    <ObjectDetectionModelContextProvider>
...
    </ObjectDetectionModelContextProvider>
  );
}

We built a new dev client for iOS on expo but it failed with this error:

❌  (node_modules/@infinitered/react-native-mlkit-object-detection/ios/RNMLKitObjectDetectionModule.swift:91:47)

  89 | 
  90 |         AsyncFunction("detectObjects") { (modelName: String, imagePath: String, promise: Promise) in
> 91 |             let logger = Logger(logHandlers: [createOSLogHandler(category: Logger.EXPO_LOG_CATEGORY)])
     |                                               ^ cannot find 'createOSLogHandler' in scope
  92 | 
  93 |             guard let objectDetector = self.objectDetectors[modelName] else { // 2. Retrieve the detector by name
  94 |                 logger.error("Model Not Found")

We then checked you releases and found that v1.0.0 is where you first added support for Expo 50 so installed that version instead. With v1.0.0, the Expo build succeeded.

We then installed the dev client on an iPhone 11 Pro with iOS v17.7 and ran the app using yarn start and then scanned the QR code. The app started up and then immediately exited.

We then tried the same thing but on a Google Pixel 4a with Android v34. On that device it seemed to hang on our splash screen.

We tried to investigate your code and found that the function useObjectDetectionModels in node_modules/@infinitered/react-native-mlkit-object-detection/build/useObjectDetectionModels.js seemed to be stuck in a recursive call to loadModels. We added this patch to this file:

diff --git a/node_modules/@infinitered/react-native-mlkit-object-detection/build/useObjectDetectionModels.js b/node_modules/@infinitered/react-native-mlkit-object-detection/build/useObjectDetectionModels.js
index 015eb93..ca9e965 100644
--- a/node_modules/@infinitered/react-native-mlkit-object-detection/build/useObjectDetectionModels.js
+++ b/node_modules/@infinitered/react-native-mlkit-object-detection/build/useObjectDetectionModels.js
@@ -56,6 +56,9 @@ function useObjectDetectionModels({ assets = {}, loadDefaultModel, defaultModelO
             const newModels = Object.fromEntries(models);
             setLoadedModels(newModels);
         }
+        if (Object.keys(loadedModels).length > 0) {
+          return
+        }
         if ((assetObjects && !assetsError) || loadDefaultModel) {
             loadModels();
         }

and that seemed to have resolved the issue.

We are reporting this issue to you in the hope that you can fix it in the latest version of your library. In the meantime, we are continuing to investigate your library to see if it meets our needs.

[asnaseer]

We have our own Tensor Flow Lite model and this library seems to work very well on iOS for object detection using this model. We call model.detectObjects("uri-to-jpeg-image") to get the results. However, on Android this always seems to return an empty array.

Code snippet:

...
const model = useObjectDetector("ourCustomModel");
...
  useEffect(() => {
    const loadModel = async () => {
      if (model && !model.isLoaded()) {
        await model.load();
      }
    };

    loadModel();
  }, [model]);
...
      if (model?.isLoaded()) {
        const detectionResult = await model!.detectObjects(imageUri);
        console.log(">>>", {
          detectionResult: JSON.stringify(detectionResult),
        });
      }
...

Have we missed some important configuration?

[lucksp]

I am unable to get results of my TFLite model, which was previously used successfully with the react-native-fast-tflite library, so I am confident the model works. Any ideas, perhaps my setup is incorrect?

const MODELS: AssetRecord = {
  // the name you'll use to refer to the model
  mfi: {
    // the relative path to the model file
    model: require('../src/assets/tflite/model.tflite'),
    options: {
      // the options you want to use for this model
      shouldEnableMultipleObjects: false,
      shouldEnableClassification: false,
      detectorMode: 'stream',
      // maxPerObjectLabelCount: 1,
    },
  },
};

 const { ObjectDetectionModelContextProvider } = useObjectDetectionModels({
    assets: MODELS,
    loadDefaultModel: false,
  });

<ObjectDetectionModelContextProvider>
                                <Stack
                                  initialRouteName="(app)/(auth)/home"
                                  screenOptions={{
                                    headerShown: false,
                                    presentation: 'fullScreenModal',
                                  }}
                                />

Then, I consume the model:

const model = useObjectDetector('mfi');
  const [modelLoaded, setModelLoaded] = useState(model?.isLoaded() ?? false);
  useEffect(() => {
    async function loadModel() {
      if (!model || modelLoaded) return;
      await model.load();
      setModelLoaded(true);
    }

    loadModel();
  }, [model, modelLoaded]);

and invoke the detection from camera photo:

const capturePhoto = async () => {
    if (!cameraRef?.current) {
      setMessage([
        'error',
        'Something went wrong. Unable to capture photo. Please reload & try again.',
      ]);
      return;
    }
    const photo = await cameraRef.current?.takePhoto({
      flash: flash,
    });

    const detectionResult = await model?.detectObjects(photo.path);
    console.log(JSON.stringify(detectionResult));

    setResult(detectionResult);
    setPhoto(photo.path);

  };

my output of the detectionResult is: [{"frame":{"origin":{"x":1064,"y":301},"size":{"y":2050,"x":2075}},"labels":[],"trackingID":0}]

So there are no labels in the result...which means it didn't recognize content of my photo.

[trevor-coleman]

There were some underlying issues with the types. I'll have to see if we can release a maintenance patch for 1.0, but in the meantime the issues will be fixed in v3.0 when it gets released.