Skip to content

Commit

Permalink
Geomodel (#724)
Browse files Browse the repository at this point in the history
* Use cv model 2.13, requires newer plugin

* Bump version and build

* Use cv model 2.13, requires newer plugin

* Bump version and build

* Use vision-plugin with speedup by taxonomy cutoff

* Bump build

* Bundle update

* Redo bundle with deleted Gemfile.lock

* Update plugin. Add option to set cutoff from JS

* Cycle through different taxonomy cutoffs

* Bump build

* Update vision plugin

* Change cycle back to one cutoff value only

* Bump build

* Show correct about text, and not missing translation

Closes #713

* Use history for going back in navigation

Closes #715
Closes #675

* Bump build

* Update vision-plugin

* Add a bit of padding to bottom of challenges screen

* Bump build

* Update with stable cv plugin

* Log timeElapsed for cv on native side

* Bump build

* Comment

* Update README.md

* Use stable vc plugin main

Technically this means we now support geomodel on iOS. We would need to add it to the app an connect to the frame processor though.

* Bump build

* Update vision-plugin

* Bump build

* Remove taxonomy cutoff override

Now it uses the plugin's inbuilt taxonomy cutoff of 0.001 the top cv score.

* Remove timesRun state

* Bump build

* Card that announces the new cv model on the home screen (#717)

* Fix vision plugin to version commit on main there

* Update Podfile.lock

* Basic version of the updates card

* Count how many species are in cv 2.13

* Looks nicer

* Bump build

* Revert back to storing only five results at a time

* Bump build

* Update vision-plugin to use new names for options

No change in Seek required

* Hardcode to never show a specific taxon

* Bump build

* Update package.json

* Remove deprecated model files; Use model on iOS directly from bundle

* Bump build

* Update fastlane

* modelFileNames TS

* Update config.example.js

* Geomodel into XCode project

* dirStorage TS, plus geomodel

* Refactor camera files helpers into own file

* Update import

* Update vision-plugin

* Prop geomodel path into camera

* Use location from species nearby provider for the geomodel
  • Loading branch information
jtklein authored Jan 8, 2025
1 parent 6add69f commit 733935e
Show file tree
Hide file tree
Showing 12 changed files with 184 additions and 113 deletions.
4 changes: 3 additions & 1 deletion components/Camera/ARCamera/ARCamera.js
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ import {
checkSavePermissions
} from "../../../utility/androidHelpers.android";
import { savePostingSuccess } from "../../../utility/loginHelpers";
import { dirModel, dirTaxonomy } from "../../../utility/dirStorage";
// TODO: this can be imported in FrameProcessorCamera directly instead of here
import { dirModel, dirGeomodel, dirTaxonomy } from "../../../utility/dirStorage";
import { createTimestamp } from "../../../utility/dateHelpers";
import { useDeviceOrientation } from "../../../utility/customHooks";
import ARCameraOverlay from "./ARCameraOverlay";
Expand Down Expand Up @@ -425,6 +426,7 @@ const ARCamera = ( ): Node => {
return (
<FrameProcessorCamera
modelPath={dirModel}
geomodelPath={dirGeomodel}
taxonomyPath={dirTaxonomy}
cameraRef={camera}
confidenceThreshold={confidenceThresholdNumber}
Expand Down
27 changes: 25 additions & 2 deletions components/Camera/ARCamera/FrameProcessorCamera.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { CameraRuntimeError } from "react-native-vision-camera";
import { Worklets } from "react-native-worklets-core";

import { useIsForeground, useDeviceOrientation } from "../../../utility/customHooks";
import { useSpeciesNearby } from "../../Providers/SpeciesNearbyProvider";
import InatVision from "./helpers/visionPluginWrapper";

import {
Expand Down Expand Up @@ -44,6 +45,7 @@ interface LogMessage {
interface Props {
cameraRef: React.RefObject<Camera>;
modelPath: string;
geomodelPath: string;
taxonomyPath: string;
confidenceThreshold: number;
filterByTaxonId: string | null;
Expand All @@ -61,6 +63,7 @@ const FrameProcessorCamera = ( props: Props ) => {
const {
cameraRef,
modelPath,
geomodelPath,
taxonomyPath,
confidenceThreshold,
filterByTaxonId,
Expand All @@ -80,6 +83,9 @@ const FrameProcessorCamera = ( props: Props ) => {

const { deviceOrientation } = useDeviceOrientation();

const { speciesNearby } = useSpeciesNearby( );
const { latitude, longitude } = speciesNearby;

const [cameraPermissionStatus, setCameraPermissionStatus] = useState( "not-determined" );
const requestCameraPermission = useCallback( async () => {
// Checking camera permission status, if granted set it and return
Expand Down Expand Up @@ -199,6 +205,14 @@ const FrameProcessorCamera = ( props: Props ) => {

const patchedRunAsync = usePatchedRunAsync();
const patchedOrientationAndroid = orientationPatchFrameProcessor( deviceOrientation );
const hasUserLocation = latitude != null && longitude != null;
// The vision-plugin has a function to look up the location of the user in a h3 gridded world
// unfortunately, I was not able to run this new function in the worklets directly,
// so we need to do this here before calling the useFrameProcessor hook.
// For predictions from file this function runs in the vision-plugin code directly.
const location = hasUserLocation
? InatVision.lookUpLocation( { latitude, longitude } )
: null;
const frameProcessor = useFrameProcessor(
( frame ) => {
"worklet";
Expand All @@ -224,7 +238,14 @@ const FrameProcessorCamera = ( props: Props ) => {
confidenceThreshold,
filterByTaxonId,
negativeFilter,
patchedOrientationAndroid
patchedOrientationAndroid,
useGeomodel: hasUserLocation,
geomodelPath,
location: {
latitude: location?.latitude,
longitude: location?.longitude,
elevation: location?.elevation
}
} );
const timeAfter = Date.now();
const timeTaken = timeAfter - timeBefore;
Expand All @@ -250,7 +271,9 @@ const FrameProcessorCamera = ( props: Props ) => {
negativeFilter,
patchedOrientationAndroid,
lastTimestamp,
fps
fps,
hasUserLocation,
location
]
);

Expand Down
3 changes: 2 additions & 1 deletion components/Splash.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import type { Node } from "react";
import styles from "../styles/splash";
import logos from "../assets/logos";
import backgrounds from "../assets/backgrounds";
import { addARCameraFiles, checkIfFirstLaunch, setCameraLaunched } from "../utility/helpers";
import { checkIfFirstLaunch, setCameraLaunched } from "../utility/helpers";
import { addARCameraFiles } from "../utility/cameraFilesHelpers";
import { setupBadges } from "../utility/badgeHelpers";
import { checkForHotStarts, checkForColdStarts, setQuickActions } from "../utility/navigationHelpers";
import { deleteFromAsyncStorage, setupUserSettings } from "../utility/settingsHelpers";
Expand Down
2 changes: 2 additions & 0 deletions config.example.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ export default {
redirectURI: "SEEK_APP_REDIRECT_URI",
// Model file names for Android and iOS
ANDROID_MODEL_FILE_NAME: "small_inception_tf1.tflite",
ANDROID_GEOMODEL_FILE_NAME: "small_geomodel_not_implemented",
ANDROID_TAXONOMY_FILE_NAME: "small_export_tax.csv",
IOS_MODEL_FILE_NAME: "small_inception_tf1.mlmodel",
IOS_GEOMODEL_FILE_NAME: "small_geomodel_not_implemented",
IOS_TAXONOMY_FILE_NAME: "small_export_tax.json"
};
2 changes: 2 additions & 0 deletions constants/modelFileNames.js → constants/modelFileNames.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ import config from "../config";

export default {
ANDROIDMODEL: config.ANDROID_MODEL_FILE_NAME,
ANDROIDGEOMODEL: config.ANDROID_GEOMODEL_FILE_NAME,
ANDROIDTAXONOMY: config.ANDROID_TAXONOMY_FILE_NAME,
IOSMODEL: `${config.IOS_MODEL_FILE_NAME}c`,
IOSGEOMODEL: `${config.IOS_GEOMODEL_FILE_NAME}c`,
IOSTAXONOMY: config.IOS_TAXONOMY_FILE_NAME
};
4 changes: 2 additions & 2 deletions ios/Podfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -1229,7 +1229,7 @@ PODS:
- VisionCamera/React (4.0.5):
- React-Core
- VisionCamera/FrameProcessors
- VisionCameraPluginInatVision (4.1.4):
- VisionCameraPluginInatVision (4.2.0):
- React-Core
- Yoga (1.14.0)

Expand Down Expand Up @@ -1649,7 +1649,7 @@ SPEC CHECKSUMS:
SDWebImageWebPCoder: e38c0a70396191361d60c092933e22c20d5b1380
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
VisionCamera: 4c1d19f1ac09f2f42f758e306fcf642536627357
VisionCameraPluginInatVision: 1bec4436cc2d2a952435ddbce2ee5457faa3ac20
VisionCameraPluginInatVision: 3028911a3e9823d9322e49c6bdcb6f0aaf57359c
Yoga: 2a16e58450c48e110211dae1159fb114bbcdcfc0

PODFILE CHECKSUM: 10e968d6ad2bca83e521f2b5695a4b3ffd1e8753
Expand Down
4 changes: 4 additions & 0 deletions ios/Seek.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
8F91A1232BD066F500513552 /* Lato-Italic.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 8F91A11B2BD066F500513552 /* Lato-Italic.ttf */; };
8F91A1242BD066F500513552 /* Lato-Medium.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 8F91A11C2BD066F500513552 /* Lato-Medium.ttf */; };
8FAB76AC2913F2E400FBEE09 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = 8FAB76AE2913F2E400FBEE09 /* InfoPlist.strings */; };
8FB8A5702D2E8C1C0002D277 /* geomodel_v2_13.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = 8FB8A56F2D2E8C1C0002D277 /* geomodel_v2_13.mlmodel */; };
/* End PBXBuildFile section */

/* Begin PBXContainerItemProxy section */
Expand Down Expand Up @@ -106,6 +107,7 @@
8FAB76C72913FE0700FBEE09 /* es-MX */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "es-MX"; path = "es-MX.lproj/InfoPlist.strings"; sourceTree = "<group>"; };
8FAB76C82913FE0800FBEE09 /* sv */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = sv; path = sv.lproj/InfoPlist.strings; sourceTree = "<group>"; };
8FAB76C92913FE0F00FBEE09 /* tr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = tr; path = tr.lproj/InfoPlist.strings; sourceTree = "<group>"; };
8FB8A56F2D2E8C1C0002D277 /* geomodel_v2_13.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; path = geomodel_v2_13.mlmodel; sourceTree = "<group>"; };
8FDFD7B629572F6D00624B83 /* bg */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = bg; path = bg.lproj/InfoPlist.strings; sourceTree = "<group>"; };
8FDFD7B72957319000624B83 /* hr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = hr; path = hr.lproj/InfoPlist.strings; sourceTree = "<group>"; };
8FDFD7B8295736DA00624B83 /* uk */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = uk; path = uk.lproj/InfoPlist.strings; sourceTree = "<group>"; };
Expand Down Expand Up @@ -222,6 +224,7 @@
AE2D56F50A4845E09545597E /* Resources */ = {
isa = PBXGroup;
children = (
8FB8A56F2D2E8C1C0002D277 /* geomodel_v2_13.mlmodel */,
8F91A1182BD066F500513552 /* Lato-Bold.ttf */,
8F91A11A2BD066F500513552 /* Lato-BoldItalic.ttf */,
8F91A11B2BD066F500513552 /* Lato-Italic.ttf */,
Expand Down Expand Up @@ -469,6 +472,7 @@
files = (
377216512880BBF300B00213 /* optimized_model_v2_13.mlmodel in Sources */,
378785D7244E5B7C00D4509C /* File.swift in Sources */,
8FB8A5702D2E8C1C0002D277 /* geomodel_v2_13.mlmodel in Sources */,
13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */,
13B07FC11A68108700A75B9A /* main.m in Sources */,
);
Expand Down
36 changes: 28 additions & 8 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@
"react-native-webview": "^11.26.1",
"react-native-worklets-core": "1.3.3",
"realm": "^12.5.1",
"vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision#8788e6d6718a4501056bad1f9ee5dbcfd354be92"
"vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision#b905ff2b9ce1cf64797d600c6cd22fc7617b2389"
},
"devDependencies": {
"@babel/core": "^7.20.0",
Expand Down
109 changes: 109 additions & 0 deletions utility/cameraFilesHelpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { Alert, Platform } from "react-native";
import RNFS from "react-native-fs";

import i18n from "../i18n";
import { dirModel, dirGeomodel, dirTaxonomy } from "./dirStorage";
import modelFiles from "../constants/modelFileNames";

const addCameraFilesAndroid = () => {
const copyFilesAndroid = ( source: string, destination: string ) => {
RNFS.copyFileAssets( source, destination ).then( ( _result ) => {
console.log( `moved file from ${source} to ${destination}` );
} ).catch( ( error ) => {
console.log( error, `error moving file from ${source} to ${destination}` );
} );
};

RNFS.readDirAssets( "camera" ).then( ( results ) => {
const model = modelFiles.ANDROIDMODEL;
const geomodel = modelFiles.ANDROIDGEOMODEL;
const taxonomy = modelFiles.ANDROIDTAXONOMY;

const hasModel = results.find( ( r ) => r.name === model );
const hasGeoModel = results.find( ( r ) => r.name === geomodel );

// Android writes over existing files
if ( hasModel !== undefined ) {
console.log( "Found model asset with filename", model );
copyFilesAndroid( `camera/${model}`, dirModel );
copyFilesAndroid( `camera/${taxonomy}`, dirTaxonomy );
} else {
console.log( "No model asset found to copy into document directory." );
Alert.alert(
i18n.t( "model.not_found_error" ),
i18n.t( "model.not_found_error_description" )
);
}
if ( hasGeoModel !== undefined ) {
console.log( "Found geomodel asset with filename", geomodel );
copyFilesAndroid( `camera/${geomodel}`, dirGeomodel );
} else {
console.log( "No geomodel asset found to copy into document directory." );
}
} );
};

const checkForModelFileIOS = () => {
RNFS.readDir( RNFS.MainBundlePath ).then( ( results ) => {
const model = modelFiles.IOSMODEL;
const hasModel = results.find( ( r ) => r.name === model );
if ( hasModel !== undefined ) {
console.log( "Found model asset with filename", model );
} else {
console.log( "No model asset found to copy into document directory." );
Alert.alert(
i18n.t( "model.not_found_error" ),
i18n.t( "model.not_found_error_description" )
);
}
} );
};

const removeDeprecatedModelFilesIOS = () => {
// On releasing cv model 2.13 (the second one ever), we changed the app to use the model
// from the main bundle directly instead of the document directory. This function removes all
// existing model files from the document directory.
RNFS.readDir( RNFS.DocumentDirectoryPath ).then( ( results ) => {
results.forEach( ( result ) => {
if ( result.name.includes( ".mlmodelc" ) || result.name.includes( "taxonomy" ) ) {
RNFS.unlink( `${RNFS.DocumentDirectoryPath}/${result.name}` ).then( () => {
console.log( "Removed deprecated model file: ", result.name );
} ).catch( ( error ) => {
console.log( error, "error removing deprecated model file" );
} );
}
} );
} );
};

const removeDeprecatedModelFilesAndroid = () => {
RNFS.readDir( RNFS.DocumentDirectoryPath ).then( ( results ) => {
results.forEach( ( result ) => {
if ( result.name === modelFiles.ANDROIDMODEL || result.name === modelFiles.ANDROIDTAXONOMY ) {
console.log( "Not removing model asset with filename", result.name );
return;
}
if ( result.name.includes( ".tflite" ) || result.name.includes( ".csv" ) ) {
RNFS.unlink( `${RNFS.DocumentDirectoryPath}/${result.name}` ).then( () => {
console.log( "Removed deprecated model file: ", result.name );
} ).catch( ( error ) => {
console.log( error, "error removing deprecated model file" );
} );
}
} );
} );
};

const addARCameraFiles = async () => {
if ( Platform.OS === "android" ) {
removeDeprecatedModelFilesAndroid();
addCameraFilesAndroid();
} else if ( Platform.OS === "ios" ) {
removeDeprecatedModelFilesIOS();
checkForModelFileIOS();
}
};

export {
addARCameraFiles
};
7 changes: 5 additions & 2 deletions utility/dirStorage.js → utility/dirStorage.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
// @flow

import { Platform } from "react-native";
import RNFS from "react-native-fs";

Expand All @@ -15,6 +13,11 @@ export const dirModel: string = Platform.select( {
android: `${RNFS.DocumentDirectoryPath}/${modelFiles.ANDROIDMODEL}`
} );

export const dirGeomodel: string = Platform.select( {
ios: `${RNFS.MainBundlePath}/${modelFiles.IOSGEOMODEL}`,
android: `${RNFS.DocumentDirectoryPath}/${modelFiles.ANDROIDGEOMODEL}`
} );

export const dirTaxonomy: string = Platform.select( {
ios: `${RNFS.MainBundlePath}/${modelFiles.IOSTAXONOMY}`,
android: `${RNFS.DocumentDirectoryPath}/${modelFiles.ANDROIDTAXONOMY}`
Expand Down
Loading

0 comments on commit 733935e

Please sign in to comment.