[WIP] onnx local test

ios/Podfile.lock

@@ -137,6 +137,14 @@ PODS:
   - objectbox_flutter_libs (0.0.1):
     - Flutter
     - ObjectBox (= 1.9.0)
+  - onnxruntime (0.0.1):
+    - Flutter
+    - onnxruntime-objc (= 1.15.1)
+  - onnxruntime-c (1.15.1)
+  - onnxruntime-objc (1.15.1):
+    - onnxruntime-objc/Core (= 1.15.1)
+  - onnxruntime-objc/Core (1.15.1):
+    - onnxruntime-c (= 1.15.1)
   - open_mail_app (0.0.1):
     - Flutter
   - OrderedSet (5.0.0)
@@ -218,6 +226,7 @@ DEPENDENCIES:
   - motionphoto (from `.symlinks/plugins/motionphoto/ios`)
   - move_to_background (from `.symlinks/plugins/move_to_background/ios`)
   - objectbox_flutter_libs (from `.symlinks/plugins/objectbox_flutter_libs/ios`)
+  - onnxruntime (from `.symlinks/plugins/onnxruntime/ios`)
   - open_mail_app (from `.symlinks/plugins/open_mail_app/ios`)
   - package_info_plus (from `.symlinks/plugins/package_info_plus/ios`)
   - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`)
@@ -249,6 +258,8 @@ SPEC REPOS:
     - Mantle
     - nanopb
     - ObjectBox
+    - onnxruntime-c
+    - onnxruntime-objc
     - OrderedSet
     - PromisesObjC
     - ReachabilitySwift
@@ -313,6 +324,8 @@ EXTERNAL SOURCES:
     :path: ".symlinks/plugins/move_to_background/ios"
   objectbox_flutter_libs:
     :path: ".symlinks/plugins/objectbox_flutter_libs/ios"
+  onnxruntime:
+    :path: ".symlinks/plugins/onnxruntime/ios"
   open_mail_app:
     :path: ".symlinks/plugins/open_mail_app/ios"
   package_info_plus:
@@ -386,6 +399,9 @@ SPEC CHECKSUMS:
   nanopb: d4d75c12cd1316f4a64e3c6963f879ecd4b5e0d5
   ObjectBox: e7ff611291a0663380e0736b46786bcd077294ff
   objectbox_flutter_libs: 0948d6feb7de4f7edaebc7a898b9e85b7fc2bc89
+  onnxruntime: e9346181d75b8dea8733bdae512a22c298962e00
+  onnxruntime-c: ebdcfd8650bcbd10121c125262f99dea681b92a3
+  onnxruntime-objc: ae7acec7a3d03eaf072d340afed7a35635c1c2a6
   open_mail_app: 794172f6a22cd16319d3ddaf45e945b2f74952b0
   OrderedSet: aaeb196f7fef5a9edf55d89760da9176ad40b93c
   package_info_plus: fd030dabf36271f146f1f3beacd48f564b0f17f7

lib/services/semantic_search/frameworks/ml_framework.dart

@@ -53,27 +53,31 @@ abstract class MLFramework {
   // ---
 
   Future<void> _initImageModel() async {
-    final path = await _getLocalImageModelPath();
-    if (File(path).existsSync()) {
-      await loadImageModel(path);
-    } else {
-      final tempFile = File(path + ".temp");
-      await _downloadFile(getImageModelRemotePath(), tempFile.path);
-      await tempFile.rename(path);
-      await loadImageModel(path);
-    }
+    //final path = await _getLocalImageModelPath();
+    const path = "assets/models/clip/clip-image-vit-32-float32.onnx";
+    await loadImageModel(path);
+    // if (File(path).existsSync()) {
+    //   await loadImageModel(path);
+    // } else {
+    //   final tempFile = File(path + ".temp");
+    //   await _downloadFile(getImageModelRemotePath(), tempFile.path);
+    //   await tempFile.rename(path);
+    //   await loadImageModel(path);
+    // }
   }
 
   Future<void> _initTextModel() async {
-    final path = await _getLocalTextModelPath();
-    if (File(path).existsSync()) {
-      await loadTextModel(path);
-    } else {
-      final tempFile = File(path + ".temp");
-      await _downloadFile(getTextModelRemotePath(), tempFile.path);
-      await tempFile.rename(path);
-      await loadTextModel(path);
-    }
+    //final path = await _getLocalTextModelPath();
+    const path = "assets/models/clip/clip-text-vit-32-float32.onnx";
+    await loadTextModel(path);
+    // if (File(path).existsSync()) {
+    //   await loadTextModel(path);
+    // } else {
+    //   final tempFile = File(path + ".temp");
+    //   await _downloadFile(getTextModelRemotePath(), tempFile.path);
+    //   await tempFile.rename(path);
+    //   await loadTextModel(path);
+    // }
   }
 
   Future<String> _getLocalImageModelPath() async {

lib/services/semantic_search/frameworks/onnx.dart

@@ -23,12 +23,12 @@ class ONNX extends MLFramework {
 
   @override
   String getImageModelRemotePath() {
-    return kModelBucketEndpoint + kImageModel;
+    return "";
   }
 
   @override
   String getTextModelRemotePath() {
-    return kModelBucketEndpoint + kTextModel;
+    return "";
   }
 
   @override
@@ -110,6 +110,7 @@ class ONNX extends MLFramework {
 class ClipImageEncoder {
   OrtSessionOptions? _sessionOptions;
   OrtSession? _session;
+  final Logger _logger = Logger("CLIPImageEncoder");
 
   ClipImageEncoder() {
     OrtEnv.instance.init();
@@ -136,7 +137,9 @@ class ClipImageEncoder {
       _session = OrtSession.fromBuffer(bytes, _sessionOptions!);
       print('image model loaded');
     } catch (e, s) {
+      _logger.severe(e, s);
       print('image model not loaded');
+      rethrow;
     }
   }
 

pubspec.lock

@@ -929,6 +929,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "0.15.3"
+  html_unescape:
+    dependency: "direct main"
+    description:
+      name: html_unescape
+      sha256: "15362d7a18f19d7b742ef8dcb811f5fd2a2df98db9f80ea393c075189e0b61e3"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.0.0"
   http:
     dependency: "direct main"
     description:
@@ -1376,6 +1384,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.0.2"
+  onnxruntime:
+    dependency: "direct main"
+    description:
+      name: onnxruntime
+      sha256: "56d0a16c24f4d1be4cf976cdc6321fb33d81515429e6e81c58d02e77d8631e0f"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.0"
   open_mail_app:
     dependency: "direct main"
     description:

pubspec.yaml

@@ -86,6 +86,7 @@ dependencies:
   fluttertoast: ^8.0.6
   freezed_annotation: ^2.2.0
   google_nav_bar: ^5.0.5
+  html_unescape: ^2.0.0
   http: ^1.1.0
   image: ^4.0.17
   image_editor: ^1.3.0
@@ -114,6 +115,7 @@ dependencies:
   # open_file: ^3.2.1
   objectbox: ^2.3.1
   objectbox_flutter_libs: any
+  onnxruntime: ^1.1.0
   open_mail_app: ^0.4.5
   package_info_plus: ^4.1.0
   page_transition: ^2.0.2
@@ -224,6 +226,7 @@ flutter:
     - assets/models/cocossd/
     - assets/models/mobilenet/
     - assets/models/scenes/
+    - assets/models/clip/
   fonts:
   - family: Inter
     fonts: