Adds example to mediapipe-task-text (#15)

* initial commit of example

* build file changes from `flutter pub get`

* update main.dart

* removes commented code

* updates example for isolates design

* Use `native-assets` to vendor MediaPipe SDK (#9)

* adding bare structure for native assets

* add MVP / first draft of build.dart

* build.dart updates

* update build.dart

TODO: stream file

* model memory troubleshooting

* vendoring script tweak

* remove development logging

* removes pointless build method

* Add utility to collect headers from google/mediapipe (#10)

* adds cmd to pull header files from google/mediapipe

* polish and missing parts from git surgery

* More comments and touch ups

* Apply suggestions from code review

* moves build command into `tool/` directory

and renames folder `build_cmd` -> `builder`

* complete build_cmd -> builder rename

* Update readme

* Added licenses

* Adds DownloadModelCommand

---------

Co-authored-by: Kate Lovett <katelovett@google.com>

* adds mediapipe_text package

* Update .vscode/settings.json

added newline

* resync headers

* regenerated core bindings

* native assets troubleshooting

this commit is broken

* Removes redundant count field

* update build.dart for correct bindings path

* download text classification model for CI

* better memory freeing in executor

* added SafeArea to example

* added CI to PRs into text package

* ci tooling change

* remove accidentally commited model

* more CI shenanigans

* lowers minimum Dart version for builder

* added smoke test for text example

* Added CI/CD for examples

* More CI tweaks

* entering "please work" territory

* d'oh

* trying more random stuff

* one more time

* it'd be funny if this helped

* more print statements

* enable reaching new print statements

* more logging

* see what's in build dir

* another test

* adding flutter config list

* turn off fail-fast for beta and master

* moar logs

* way moar prints

* moare things

* commit rest of rename

* moar whatevers

* adds manifest files generated by new sdks_finder command

* adds sdks_finder command to builder utility

* propagates changes to existing commands

* updates in response to code review

* updates to build.dart and tests

* add Android runtime

* sdks_finder logging improvement for when build folders change names

* refreshed symbols from google/mediapipe

* cleanup

* loosens closeness thresholds in integration tests

* separate build commands for macos architectures

* restores fail-fast setting to CI

* removed stale logging statements from CI

* removes accidentally committed lines

* add formatting of sdk_downloads.dart for CI

* fixes broken example test

* code touch ups from @Piinks code review

---------

Co-authored-by: Kate Lovett <katelovett@google.com>

.github/workflows/main.yaml

@@ -7,7 +7,7 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main, ffi-wrapper, ffi-wrapper-core-pkg]
+    branches: [main, ffi-wrapper, ffi-wrapper-text-pkg]
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * *" # Every day at midnight
@@ -25,7 +25,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        flutter_version: [stable, beta, master]
+        flutter_version: [master]
+        # TODO(craiglabenz): reverse once CI works
+        # flutter_version: [stable, beta, master]
         # TODO(craiglabenz): Add `ubuntu-latest` and `windows-latest` when those artifacts exist
         os: [macos-latest]
     steps:

Makefile

@@ -5,10 +5,10 @@ headers:
 # Runs `ffigen` for all packages
 generate: generate_core generate_text
 
-# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests
+# Runs `ffigen` for all packages and runs all tests
 test: generate_text test_text generate_core test_core
 
-# Runs `ffigen` for all packages and all tests for all packages
+# Runs all tests for all packages
 test_only: test_core test_text
 
 # Runs `sdks_finder` to update manifest files
@@ -17,9 +17,17 @@ sdks:
 
 # Rebuilds the MediaPipe task for macOS
 # Assumes google/mediapipe and google/flutter-mediapipe are siblings on the file system
-compile_text_classifier_macos:
+compile_text_classifier_macos_arm:
 	cd ../mediapipe && bazel build --linkopt -s --config darwin_arm64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
-	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_arm64.dylib
+
+compile_text_classifier_macos_x86:
+	cd ../mediapipe && bazel build --linkopt -s --config darwin_x86_64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_x64.dylib
+
+# Runs `sdks_finder` to update manifest files
+sdks:
+	dart tool/builder/bin/main.dart sdks
 
 # Core ---
 
@@ -37,9 +45,10 @@ test_core:
 generate_text:
 	cd packages/mediapipe-task-text && dart --enable-experiment=native-assets run ffigen --config=ffigen.yaml
 
-# Runs `ffigen` for `mediapipe_text` and all text tests
-test_text: compile_fake_text test_text_only
-
 # Runs all text tests
-test_text_only:
+test_text:
 	cd packages/mediapipe-task-text && flutter test
+	cd packages/mediapipe-task-text/example && flutter test
+
+example_text:
+	cd packages/mediapipe-task-text/example && flutter run -d macos

packages/mediapipe-core/lib/generated/core_symbols.yaml

@@ -6,6 +6,8 @@ files:
     symbols:
       c:@S@BaseOptions:
         name: BaseOptions
+      c:@S@Categories:
+        name: Categories
       c:@S@Category:
         name: Category
       c:@S@ClassificationResult:

packages/mediapipe-core/lib/src/containers.dart

@@ -40,19 +40,19 @@ class Category {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Category] instances.
-  static List<Category> fromStructs(
+  static List<Category> structsToDart(
     Pointer<bindings.Category> structs,
     int count,
   ) {
     final categories = <Category>[];
     for (int i = 0; i < count; i++) {
-      categories.add(fromStruct(structs[i]));
+      categories.add(structToDart(structs[i]));
     }
     return categories;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance.
-  static Category fromStruct(bindings.Category struct) {
+  static Category structToDart(bindings.Category struct) {
     return Category(
       index: struct.index,
       score: struct.score,
@@ -62,19 +62,25 @@ class Category {
   }
 
   /// Releases all C memory associated with a list of [bindings.Category] pointers.
-  /// This method is important to call after calling [Category.fromStructs] to
+  /// This method is important to call after calling [Category.structsToDart] to
   /// convert that C memory into pure-Dart objects.
   static void freeStructs(Pointer<bindings.Category> structs, int count) {
     int index = 0;
     while (index < count) {
       bindings.Category obj = structs[index];
-      calloc.free(obj.category_name);
-      calloc.free(obj.display_name);
+      Category.freeStruct(obj);
       index++;
     }
     calloc.free(structs);
   }
 
+  /// Releases the string fields associated with a single [bindings.Category]
+  /// struct.
+  static void freeStruct(bindings.Category struct) {
+    calloc.free(struct.category_name);
+    calloc.free(struct.display_name);
+  }
+
   @override
   String toString() => 'Category(index=$index, score=$score, '
       'categoryName=$categoryName, displayName=$displayName)';
@@ -108,22 +114,22 @@ class Classifications {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Classifications] instances.
-  static List<Classifications> fromStructs(
+  static List<Classifications> structsToDart(
     Pointer<bindings.Classifications> structs,
     int count,
   ) {
     final classifications = <Classifications>[];
     for (int i = 0; i < count; i++) {
-      classifications.add(fromStruct(structs[i]));
+      classifications.add(structToDart(structs[i]));
     }
     return classifications;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications]
   /// instance.
-  static Classifications fromStruct(bindings.Classifications struct) {
+  static Classifications structToDart(bindings.Classifications struct) {
     return Classifications(
-      categories: Category.fromStructs(
+      categories: Category.structsToDart(
         struct.categories,
         struct.categories_count,
       ),
@@ -133,7 +139,7 @@ class Classifications {
   }
 
   /// Releases all C memory associated with a list of [bindings.Classifications]
-  /// pointers. This method is important to call after calling [Classifications.fromStructs]
+  /// pointers. This method is important to call after calling [Classifications.structsToDart]
   /// to convert that C memory into pure-Dart objects.
   static void freeStructs(
     Pointer<bindings.Classifications> structs,

packages/mediapipe-core/lib/src/task_options.dart

@@ -21,51 +21,89 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart'
 ///    classifier's desired behavior.
 class BaseOptions extends Equatable {
   /// Generative constructor that creates a [BaseOptions] instance.
-  const BaseOptions({this.modelAssetBuffer, this.modelAssetPath})
-      : assert(
+  const BaseOptions._({
+    this.modelAssetBuffer,
+    this.modelAssetPath,
+    required _BaseOptionsType type,
+  })  : assert(
           !(modelAssetBuffer == null && modelAssetPath == null),
           'You must supply either `modelAssetBuffer` or `modelAssetPath`',
         ),
         assert(
           !(modelAssetBuffer != null && modelAssetPath != null),
           'You must only supply one of `modelAssetBuffer` and `modelAssetPath`',
-        );
+        ),
+        _type = type;
+
+  /// Constructor for [BaseOptions] classes using a file system path.
+  ///
+  /// In practice, this is unsupported, as assets in Flutter are bundled into
+  /// the build output and not available on disk. However, it can potentially
+  /// be helpful for testing / development purposes.
+  factory BaseOptions.path(String path) => BaseOptions._(
+        modelAssetPath: path,
+        type: _BaseOptionsType.path,
+      );
+
+  /// Constructor for [BaseOptions] classes using an in-memory pointer to the
+  /// MediaPipe SDK.
+  ///
+  /// In practice, this is the only option supported for production builds.
+  factory BaseOptions.memory(Uint8List buffer) {
+    return BaseOptions._(
+      modelAssetBuffer: buffer,
+      type: _BaseOptionsType.memory,
+    );
+  }
 
   /// The model asset file contents as bytes;
   final Uint8List? modelAssetBuffer;
 
   /// Path to the model asset file.
   final String? modelAssetPath;
 
+  final _BaseOptionsType _type;
+
   /// Converts this pure-Dart representation into C-memory suitable for the
   /// MediaPipe SDK to instantiate various classifiers.
   Pointer<bindings.BaseOptions> toStruct() {
     final struct = calloc<bindings.BaseOptions>();
 
-    if (modelAssetPath != null) {
-      struct.ref.model_asset_path = prepareString(modelAssetPath!);
-    }
-    if (modelAssetBuffer != null) {
-      struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
-      struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes;
+    switch (_type) {
+      case _BaseOptionsType.path:
+        {
+          struct.ref.model_asset_path = prepareString(modelAssetPath!);
+        }
+      case _BaseOptionsType.memory:
+        {
+          struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
+          struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes;
+        }
     }
+
     return struct;
   }
 
-  @override
-  List<Object?> get props => [modelAssetBuffer, modelAssetPath];
-
   /// Releases all C memory held by this [bindings.BaseOptions] struct.
   static void freeStruct(bindings.BaseOptions struct) {
-    if (struct.model_asset_buffer.address != 0) {
-      calloc.free(struct.model_asset_buffer);
-    }
-    if (struct.model_asset_path.address != 0) {
+    if (struct.model_asset_path.isNotNullPointer) {
       calloc.free(struct.model_asset_path);
     }
+    if (struct.model_asset_buffer.isNotNullPointer) {
+      calloc.free(struct.model_asset_buffer);
+    }
   }
+
+  @override
+  List<Object?> get props => [
+        modelAssetBuffer,
+        modelAssetPath,
+        modelAssetBuffer?.lengthInBytes,
+      ];
 }
 
+enum _BaseOptionsType { path, memory }
+
 /// Dart representation of MediaPipe's "ClassifierOptions" concept.
 ///
 /// Classifier options shared across MediaPipe classification tasks.

packages/mediapipe-core/lib/src/test_utils.dart

@@ -0,0 +1,54 @@
+import 'dart:ffi';
+import 'package:ffi/ffi.dart';
+
+import 'package:mediapipe_core/mediapipe_core.dart';
+import 'package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart'
+    as core_bindings;
+
+/// Hydrates a faked [core_bindings.Category] object.
+void populateCategory(
+  core_bindings.Category category, {
+  int index = 1,
+  double score = 0.9,
+  String? categoryName = 'Positive',
+  String? displayName,
+}) {
+  category.index = index;
+  category.score = score;
+
+  if (categoryName != null) {
+    category.category_name = prepareString(categoryName);
+  }
+  if (displayName != null) {
+    category.display_name = prepareString(displayName);
+  }
+}
+
+/// Hydrates a faked [core_bindings.Classifications] object.
+///
+/// If [categories] is passed, [numCategories] must indicate its length;
+/// otherwise, this function generates [numCategories] faked
+/// [core_bindings.Category] objects.
+void populateClassifications(
+  core_bindings.Classifications classifications, {
+  Pointer<core_bindings.Category>? categories,
+  int numCategories = 2,
+  int headIndex = 1,
+  String headName = 'Head',
+}) {
+  if (isNotNullOrNullPointer(categories)) {
+    classifications.categories = categories!;
+    classifications.categories_count = numCategories;
+  } else {
+    final ptrs = calloc<core_bindings.Category>(numCategories);
+    int count = 0;
+    while (count < numCategories) {
+      populateCategory(ptrs[count]);
+      count++;
+    }
+    classifications.categories = ptrs;
+    classifications.categories_count = numCategories;
+  }
+  classifications.head_name = prepareString(headName);
+  classifications.head_index = headIndex;
+}

packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart

@@ -22,10 +22,10 @@ import 'dart:ffi' as ffi;
 final class BaseOptions extends ffi.Struct {
   external ffi.Pointer<ffi.Char> model_asset_buffer;
 
-  external ffi.Pointer<ffi.Char> model_asset_path;
-
   @ffi.UnsignedInt()
   external int model_asset_buffer_count;
+
+  external ffi.Pointer<ffi.Char> model_asset_path;
 }
 
 final class __mbstate_t extends ffi.Union {
@@ -152,6 +152,13 @@ final class Category extends ffi.Struct {
   external ffi.Pointer<ffi.Char> display_name;
 }
 
+final class Categories extends ffi.Struct {
+  external ffi.Pointer<Category> categories;
+
+  @ffi.Uint32()
+  external int categories_count;
+}
+
 final class Classifications extends ffi.Struct {
   external ffi.Pointer<Category> categories;