From 2ebbc325cf73b5183fa19e426f1525a9fa2bf48c Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 08:34:18 -0700 Subject: [PATCH] adds cmd to pull header files from google/mediapipe --- packages/build_cmd/.gitignore | 3 + packages/build_cmd/CHANGELOG.md | 3 + packages/build_cmd/README.md | 2 + packages/build_cmd/analysis_options.yaml | 30 ++++ packages/build_cmd/bin/main.dart | 10 ++ .../build_cmd/lib/generated/core_symbols.yaml | 38 ++++ packages/build_cmd/lib/repo_finder.dart | 78 +++++++++ packages/build_cmd/lib/sync_headers.dart | 163 ++++++++++++++++++ packages/build_cmd/pubspec.yaml | 20 +++ 9 files changed, 347 insertions(+) create mode 100644 packages/build_cmd/.gitignore create mode 100644 packages/build_cmd/CHANGELOG.md create mode 100644 packages/build_cmd/README.md create mode 100644 packages/build_cmd/analysis_options.yaml create mode 100644 packages/build_cmd/bin/main.dart create mode 100644 packages/build_cmd/lib/generated/core_symbols.yaml create mode 100644 packages/build_cmd/lib/repo_finder.dart create mode 100644 packages/build_cmd/lib/sync_headers.dart create mode 100644 packages/build_cmd/pubspec.yaml diff --git a/packages/build_cmd/.gitignore b/packages/build_cmd/.gitignore new file mode 100644 index 00000000..3a857904 --- /dev/null +++ b/packages/build_cmd/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/packages/build_cmd/CHANGELOG.md b/packages/build_cmd/CHANGELOG.md new file mode 100644 index 00000000..effe43c8 --- /dev/null +++ b/packages/build_cmd/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/packages/build_cmd/README.md b/packages/build_cmd/README.md new file mode 100644 index 00000000..3816eca3 --- /dev/null +++ b/packages/build_cmd/README.md @@ -0,0 +1,2 @@ +A sample command-line application with an entrypoint in `bin/`, library code +in `lib/`, and example unit test in `test/`. diff --git a/packages/build_cmd/analysis_options.yaml b/packages/build_cmd/analysis_options.yaml new file mode 100644 index 00000000..dee8927a --- /dev/null +++ b/packages/build_cmd/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/packages/build_cmd/bin/main.dart b/packages/build_cmd/bin/main.dart new file mode 100644 index 00000000..73f73896 --- /dev/null +++ b/packages/build_cmd/bin/main.dart @@ -0,0 +1,10 @@ +import 'package:args/command_runner.dart'; +import 'package:build_cmd/sync_headers.dart'; + +final runner = CommandRunner( + 'build', + 'Performs build operations for google/flutter-mediapipe that ' + 'depend on contents in this repository', +)..addCommand(SyncHeadersCommand()); + +void main(List arguments) => runner.run(arguments); diff --git a/packages/build_cmd/lib/generated/core_symbols.yaml b/packages/build_cmd/lib/generated/core_symbols.yaml new file mode 100644 index 00000000..29377929 --- /dev/null +++ b/packages/build_cmd/lib/generated/core_symbols.yaml @@ -0,0 +1,38 @@ +format_version: 1.0.0 +files: + package:build/src/mediapipe_common_bindings.dart: + used-config: + ffi-native: false + symbols: + c:@S@BaseOptions: + name: BaseOptions + c:@S@Category: + name: Category + c:@S@ClassificationResult: + name: ClassificationResult + c:@S@Classifications: + name: Classifications + c:@S@ClassifierOptions: + name: ClassifierOptions + c:@S@__darwin_pthread_handler_rec: + name: __darwin_pthread_handler_rec + c:@S@_opaque_pthread_attr_t: + name: _opaque_pthread_attr_t + c:@S@_opaque_pthread_cond_t: + name: _opaque_pthread_cond_t + c:@S@_opaque_pthread_condattr_t: + name: _opaque_pthread_condattr_t + c:@S@_opaque_pthread_mutex_t: + name: _opaque_pthread_mutex_t + c:@S@_opaque_pthread_mutexattr_t: + name: _opaque_pthread_mutexattr_t + c:@S@_opaque_pthread_once_t: + name: _opaque_pthread_once_t + c:@S@_opaque_pthread_rwlock_t: + name: _opaque_pthread_rwlock_t + c:@S@_opaque_pthread_rwlockattr_t: + name: _opaque_pthread_rwlockattr_t + c:@S@_opaque_pthread_t: + name: _opaque_pthread_t + c:@UA@__mbstate_t: + name: __mbstate_t diff --git a/packages/build_cmd/lib/repo_finder.dart b/packages/build_cmd/lib/repo_finder.dart new file mode 100644 index 00000000..92bba55e --- /dev/null +++ b/packages/build_cmd/lib/repo_finder.dart @@ -0,0 +1,78 @@ +import 'dart:io' as io; +import 'package:args/args.dart'; +import 'package:args/command_runner.dart'; +import 'package:path/path.dart' as path; +import 'package:io/ansi.dart'; + +mixin RepoFinderMixin on Command { + void addSourceOption(ArgParser argParser) { + argParser.addOption( + 'source', + abbr: 's', + help: 'The location of google/mediapipe. Defaults to being ' + 'adjacent to google/flutter-mediapipe.', + ); + } + + /// Looks upward for the root of the `google/mediapipe` repository. This assumes + /// the `dart build` command is executed from within said repository. If it is + /// not executed from within, then this searching algorithm will reach the root + /// of the file system, log the error, and exit. + io.Directory findFlutterMediaPipeRoot() { + io.Directory dir = io.Directory(path.current); + while (true) { + if (_isFlutterMediaPipeRoot(dir)) { + return dir; + } + dir = dir.parent; + if (dir.parent.path == dir.path) { + io.stderr.writeln( + wrapWith( + 'Failed to find google/mediapipe root directory. ' + 'Did you execute this command from within the repository?', + [red], + ), + ); + io.exit(1); + } + } + } + + /// Finds the `google/mediapipe` checkout where artifacts built in this + /// repository should be sourced. By default, this command assumes the two + /// repositories are siblings on the file system, but the `--origin` flag + /// allows for this assumption to be overridden. + io.Directory findMediaPipeRoot( + io.Directory flutterMediaPipeDir, + String? origin, + ) { + final flutterMediaPipeDirectory = io.Directory( + origin ?? + path.joinAll([ + flutterMediaPipeDir.parent.absolute.path, + 'mediapipe', + ]), + ); + + if (!flutterMediaPipeDirectory.existsSync()) { + io.stderr.writeln( + 'Could not find ${flutterMediaPipeDirectory.absolute.path}. ' + 'Folder does not exist.', + ); + io.exit(1); + } + return flutterMediaPipeDirectory; + } +} + +/// Looks for the sentinel file of this repository's root directory. This allows +/// the `dart build` command to be run from various locations within the +/// `google/mediapipe` repository and still correctly set paths for all of its +/// operations. +bool _isFlutterMediaPipeRoot(io.Directory dir) { + return io.File( + path.joinAll( + [dir.absolute.path, '.flutter-mediapipe-root'], + ), + ).existsSync(); +} diff --git a/packages/build_cmd/lib/sync_headers.dart b/packages/build_cmd/lib/sync_headers.dart new file mode 100644 index 00000000..94dfa4b7 --- /dev/null +++ b/packages/build_cmd/lib/sync_headers.dart @@ -0,0 +1,163 @@ +import 'dart:convert'; +import 'dart:io' as io; +import 'package:args/command_runner.dart'; +import 'package:build_cmd/repo_finder.dart'; +import 'package:io/ansi.dart'; +import 'package:path/path.dart' as path; +import 'package:process/process.dart'; + +/// Relative header paths (in both repositories) +final containers = 'mediapipe/tasks/c/components/containers'; +final processors = 'mediapipe/tasks/c/components/processors'; +final core = 'mediapipe/tasks/c/core'; +final tc = 'mediapipe/tasks/c/text/text_classifier'; + +/// google/flutter-mediapipe package paths +final corePackage = 'packages/mediapipe-core/third_party'; +final textPackage = 'packages/mediapipe-task-text/third_party'; + +/// First string is its relative location in both repositories, +/// Second string is its package location in `google/flutter-mediapipe`, +/// Third string is the file name +/// Fourth param is an optional function to modify the file +List<(String, String, String, Function(io.File)?)> headerPaths = [ + (containers, corePackage, 'category.h', null), + (containers, corePackage, 'classification_result.h', null), + (core, corePackage, 'base_options.h', null), + (processors, corePackage, 'classifier_options.h', null), + (tc, textPackage, 'text_classifier.h', relativeIncludes), +]; + +class SyncHeadersCommand extends Command with RepoFinderMixin { + @override + String description = 'Syncs header files to google/flutter-mediapipe'; + @override + String name = 'headers'; + + SyncHeadersCommand() { + argParser.addFlag( + 'overwrite', + abbr: 'o', + defaultsTo: true, + help: 'If true, will overwrite existing header files ' + 'at destination locations.', + ); + addSourceOption(argParser); + } + + @override + Future run() async { + final io.Directory flutterMediaPipeDirectory = findFlutterMediaPipeRoot(); + final io.Directory mediaPipeDirectory = findMediaPipeRoot( + flutterMediaPipeDirectory, + argResults!['source'], + ); + + final config = Options( + allowOverwrite: argResults!['overwrite'], + mediaPipeDir: mediaPipeDirectory, + flutterMediaPipeDir: flutterMediaPipeDirectory, + ); + + await copyHeaders(config); + } + + Future copyHeaders(Options config) async { + final mgr = LocalProcessManager(); + for (final tup in headerPaths) { + final headerFile = io.File(path.joinAll( + [config.mediaPipeDir.absolute.path, tup.$1, tup.$3], + )); + if (!headerFile.existsSync()) { + io.stderr.writeln( + 'Expected to find ${headerFile.path}, but ' + 'file does not exist.', + ); + io.exit(1); + } + final destinationPath = path.joinAll( + [config.flutterMediaPipeDir.absolute.path, tup.$2, tup.$1, tup.$3], + ); + final destinationFile = io.File(destinationPath); + if (destinationFile.existsSync() && !config.allowOverwrite) { + io.stdout.writeln( + 'Warning: Not overwriting existing file at $destinationPath. ' + 'Skipping ${tup.$3}.', + ); + continue; + } + + ensureFolders(io.File(destinationPath)); + + final process = await mgr.start([ + 'cp', + headerFile.path, + destinationPath, + ], runInShell: true); + int processExitCode = await process.exitCode; + if (processExitCode != 0) { + final processStdErr = utf8.decoder.convert( + (await process.stderr.toList()) + .fold>([], (arr, el) => arr..addAll(el))); + io.stderr.write(wrapWith(processStdErr, [red])); + + final processStdOut = utf8.decoder.convert( + (await process.stdout.toList()) + .fold>([], (arr, el) => arr..addAll(el))); + io.stderr.write(wrapWith(processStdOut, [red])); + io.exit(processExitCode); + } else { + io.stderr.writeln(wrapWith('Copied ${tup.$3}', [green])); + } + + // Call the final modification function, if supplied + if (tup.$4 != null) { + tup.$4!.call(destinationFile); + } + } + } + + /// Builds any missing folders between the file and the root of the repository + void ensureFolders(io.File file) { + io.Directory parent = file.parent; + List dirsToCreate = []; + while (!parent.existsSync()) { + dirsToCreate.add(parent); + parent = parent.parent; + } + for (io.Directory dir in dirsToCreate.reversed) { + dir.createSync(); + } + } +} + +class Options { + const Options({ + required this.allowOverwrite, + required this.mediaPipeDir, + required this.flutterMediaPipeDir, + }); + + final bool allowOverwrite; + final io.Directory mediaPipeDir; + final io.Directory flutterMediaPipeDir; +} + +void relativeIncludes(io.File textClassifierHeader) { + assert(textClassifierHeader.path.endsWith('text_classifier.h')); + String contents = textClassifierHeader.readAsStringSync(); + + Map rewrites = { + 'mediapipe/tasks/c/components/containers/classification_result.h': + '../../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h', + 'mediapipe/tasks/c/components/processors/classifier_options.h': + '../../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h', + 'mediapipe/tasks/c/core/base_options.h': + '../../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h', + }; + + for (final rewrite in rewrites.entries) { + contents = contents.replaceAll(rewrite.key, rewrite.value); + } + textClassifierHeader.writeAsStringSync(contents); +} diff --git a/packages/build_cmd/pubspec.yaml b/packages/build_cmd/pubspec.yaml new file mode 100644 index 00000000..3bed0686 --- /dev/null +++ b/packages/build_cmd/pubspec.yaml @@ -0,0 +1,20 @@ +name: build +description: Performs build operations for google/flutter-mediapipe that depend + on contents in this repository. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo +environment: + sdk: ^3.2.0-162.0.dev + +# Add regular dependencies here. +dependencies: + args: ^2.4.2 + io: ^1.0.4 + logging: ^1.2.0 + path: ^1.8.0 + process: ^5.0.0 + +dev_dependencies: + ffigen: ^9.0.1 + lints: ^2.1.0 + test: ^1.24.0