Skip to content

Commit 551b463

Browse files
craiglabenzPiinks
craiglabenz
and
Piinks
committed
Add utility to collect headers from google/mediapipe (#10)
* adds cmd to pull header files from google/mediapipe * polish and missing parts from git surgery * More comments and touch ups * Apply suggestions from code review * moves build command into `tool/` directory and renames folder `build_cmd` -> `builder` * complete build_cmd -> builder rename * Update readme * Added licenses * Adds DownloadModelCommand --------- Co-authored-by: Kate Lovett <katelovett@google.com>
1 parent 6c61f3d commit 551b463

File tree

11 files changed

+507
-0
lines changed

11 files changed

+507
-0
lines changed

‎.flutter-mediapipe-root‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// Used to normalize the paths of commands.
2+
// The contents of this file do not matter.

‎packages/mediapipe-task-text/example/assets/libtext_classifier.dylib‎

36.8 MB
Binary file not shown.

‎tool/builder/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# https://dart.dev/guides/libraries/private-files
2+
# Created by `dart pub`
3+
.dart_tool/

‎tool/builder/CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
## 1.0.0
2+
3+
- Initial version with headers command to sync C headers from `google/mediapipe`.

‎tool/builder/README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Flutter MediaPipe builder
2+
3+
Helper utility which performs build or CI/CD step operations necessary to develop, release, and use `flutter-mediapipe`.
4+
5+
### Usage:
6+
7+
Usage depends on which task you need to accomplish. All supported workflows are described below.
8+
9+
#### Header aggregation
10+
11+
Header files across all tasks in `google/flutter-mediapipe` have to stay in sync with their origin, `google/mediapipe`. To
12+
resync these files, check out both repositories on the same machine (ideally next to each other on the file system) and run:
13+
14+
```sh
15+
$ dart tool/builder/bin/main.dart headers
16+
```
17+
18+
--
19+
20+
Check back in the future for any additional development workflows this command may support.

‎tool/builder/analysis_options.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
include: package:lints/recommended.yaml
2+

‎tool/builder/bin/main.dart‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io' as io;
6+
import 'package:args/command_runner.dart';
7+
import 'package:builder/download_model.dart';
8+
import 'package:builder/sync_headers.dart';
9+
import 'package:logging/logging.dart';
10+
11+
final runner = CommandRunner(
12+
'build',
13+
'Performs build operations for google/flutter-mediapipe that '
14+
'depend on contents in this repository',
15+
)
16+
..addCommand(SyncHeadersCommand())
17+
..addCommand(DownloadModelCommand());
18+
19+
void main(List<String> arguments) {
20+
Logger.root.onRecord.listen((LogRecord record) {
21+
io.stdout
22+
.writeln('${record.level.name}: ${record.time}: ${record.message}');
23+
});
24+
runner.run(arguments);
25+
}

‎tool/builder/lib/download_model.dart‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io' as io;
6+
import 'package:args/command_runner.dart';
7+
import 'package:builder/repo_finder.dart';
8+
import 'package:http/http.dart' as http;
9+
import 'package:logging/logging.dart';
10+
import 'package:path/path.dart' as path;
11+
12+
final _log = Logger('DownloadModelCommand');
13+
14+
enum Models {
15+
textclassification,
16+
languagedetection,
17+
}
18+
19+
class DownloadModelCommand extends Command with RepoFinderMixin {
20+
@override
21+
String description = 'Downloads a given MediaPipe model and places it in '
22+
'the designated location.';
23+
@override
24+
String name = 'model';
25+
26+
DownloadModelCommand() {
27+
argParser
28+
..addOption(
29+
'model',
30+
abbr: 'm',
31+
allowed: [
32+
// Values will be added to this as the repository gets more
33+
// integration tests that require new models.
34+
Models.textclassification.name,
35+
Models.languagedetection.name,
36+
],
37+
help: 'The desired model to download. Use this option if you want the '
38+
'standard model for a given task. Using this option also removes any '
39+
'need to use the `destination` option, as a value here implies a '
40+
'destination. However, you still can specify a destination to '
41+
'override the default location where the model is placed.\n'
42+
'\n'
43+
'Note: Either this or `custommodel` must be used. If both are '
44+
'supplied, `model` is used.',
45+
)
46+
..addOption(
47+
'custommodel',
48+
abbr: 'c',
49+
help: 'The desired model to download. Use this option if you want to '
50+
'specify a specific and nonstandard model. Using this option means '
51+
'you *must* use the `destination` option.\n'
52+
'\n'
53+
'Note: Either this or `model` must be used. If both are supplied, '
54+
'`model` is used.',
55+
)
56+
..addOption(
57+
'destination',
58+
abbr: 'd',
59+
help:
60+
'The location to place the downloaded model. This value is required '
61+
'if you use the `custommodel` option, but optional if you use the '
62+
'`model` option.',
63+
);
64+
}
65+
66+
static final Map<String, String> _standardModelSources = {
67+
Models.textclassification.name:
68+
'https://storage.googleapis.com/mediapipe-models/text_classifier/bert_classifier/float32/1/bert_classifier.tflite',
69+
Models.languagedetection.name:
70+
'https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite',
71+
};
72+
73+
static final Map<String, String> _standardModelDestinations = {
74+
Models.textclassification.name:
75+
'packages/mediapipe-task-text/example/assets/',
76+
Models.languagedetection.name:
77+
'packages/mediapipe-task-text/example/assets/',
78+
};
79+
80+
@override
81+
Future<void> run() async {
82+
final io.Directory flutterMediaPipeDirectory = findFlutterMediaPipeRoot();
83+
84+
late final String modelSource;
85+
late final String modelDestination;
86+
87+
if (argResults!['model'] != null) {
88+
modelSource = _standardModelSources[argResults!['model']]!;
89+
modelDestination = (_isArgProvided(argResults!['destination']))
90+
? argResults!['destination']
91+
: _standardModelDestinations[argResults!['model']]!;
92+
} else {
93+
if (argResults!['custommodel'] == null) {
94+
throw Exception(
95+
'You must use either the `model` or `custommodel` option.',
96+
);
97+
}
98+
if (argResults!['destination'] == null) {
99+
throw Exception(
100+
'If you do not use the `model` option, then you must supply a '
101+
'`destination`, as a "standard" destination cannot be used.',
102+
);
103+
}
104+
modelSource = argResults!['custommodel'];
105+
modelDestination = argResults!['destination'];
106+
}
107+
108+
io.File destinationFile = io.File(
109+
path.joinAll([
110+
flutterMediaPipeDirectory.absolute.path,
111+
modelDestination,
112+
modelSource.split('/').last,
113+
]),
114+
);
115+
ensureFolders(destinationFile);
116+
await downloadModel(modelSource, destinationFile);
117+
}
118+
119+
Future<void> downloadModel(
120+
String modelSource,
121+
io.File destinationFile,
122+
) async {
123+
_log.info('Downloading $modelSource');
124+
125+
// TODO(craiglabenz): Convert to StreamedResponse
126+
final response = await http.get(Uri.parse(modelSource));
127+
128+
if (response.statusCode != 200) {
129+
throw Exception('${response.statusCode} ${response.reasonPhrase} :: '
130+
'$modelSource');
131+
}
132+
133+
if (!(await destinationFile.exists())) {
134+
_log.fine('Creating file at ${destinationFile.absolute.path}');
135+
await destinationFile.create();
136+
}
137+
138+
_log.fine('Downloaded ${response.contentLength} bytes');
139+
_log.info('Saving to ${destinationFile.absolute.path}');
140+
await destinationFile.writeAsBytes(response.bodyBytes);
141+
}
142+
}
143+
144+
bool _isArgProvided(String? val) => val != null && val != '';

‎tool/builder/lib/repo_finder.dart‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io' as io;
6+
import 'package:args/args.dart';
7+
import 'package:args/command_runner.dart';
8+
import 'package:path/path.dart' as path;
9+
import 'package:io/ansi.dart';
10+
11+
/// Mixin to help [Command] subclasses locate both `google/mediapipe` and
12+
/// the root of `google/flutter-mediapipe` (this repository).
13+
///
14+
/// The primary methods are [findFlutterMediaPipeRoot] and [findMediaPipeRoot].
15+
///
16+
/// By default, the root for `google/flutter-mediapipe` is determined by the
17+
/// firest ancestor directory which contains a `.flutter-mediapipe-root` file
18+
/// (whose contents are irrelevant), and the root of `google/mediapipe` is
19+
/// expected to be a sibling of that. However, the `--source` flag can overwrite
20+
/// this expectation and specify an absolute path where to find `google/mediapipe`.
21+
///
22+
/// Note that it is not possible to override the method of locating the root of
23+
/// `google/flutter-mediapipe`.
24+
mixin RepoFinderMixin on Command {
25+
/// Name of the file which, when found, indicates the root of this repository.
26+
static String sentinelFileName = '.flutter-mediapipe-root';
27+
28+
void addSourceOption(ArgParser argParser) {
29+
argParser.addOption(
30+
'source',
31+
abbr: 's',
32+
help: 'The location of google/mediapipe. Defaults to being '
33+
'adjacent to google/flutter-mediapipe.',
34+
);
35+
}
36+
37+
/// Looks upward for the root of the `google/mediapipe` repository. This assumes
38+
/// the `dart build` command is executed from within said repository. If it is
39+
/// not executed from within, then this searching algorithm will reach the root
40+
/// of the file system, log the error, and exit.
41+
io.Directory findFlutterMediaPipeRoot() {
42+
final placesChecked = <io.Directory>[];
43+
io.Directory dir = io.Directory(path.current);
44+
while (true) {
45+
if (_isFlutterMediaPipeRoot(dir)) {
46+
return dir;
47+
}
48+
placesChecked.add(dir);
49+
dir = dir.parent;
50+
if (dir.parent.path == dir.path) {
51+
io.stderr.writeln(
52+
wrapWith(
53+
'Failed to find google/flutter-mediapipe root directory. '
54+
'Did you execute this command from within the repository?\n'
55+
'Looked in:',
56+
[red],
57+
),
58+
);
59+
io.stderr.writeln(
60+
wrapWith(
61+
placesChecked
62+
.map<String>((dir) => ' - ${dir.absolute.path}')
63+
.toList()
64+
.join('\n'),
65+
[red],
66+
),
67+
);
68+
io.exit(1);
69+
}
70+
}
71+
}
72+
73+
/// Finds the `google/mediapipe` checkout where artifacts built in this
74+
/// repository should be sourced. By default, this command assumes the two
75+
/// repositories are siblings on the file system, but the `--source` flag
76+
/// allows for this assumption to be overridden.
77+
io.Directory findMediaPipeRoot(
78+
io.Directory flutterMediaPipeDir,
79+
String? source,
80+
) {
81+
final mediaPipeDirectory = io.Directory(
82+
source ??
83+
path.joinAll([flutterMediaPipeDir.parent.absolute.path, 'mediapipe']),
84+
);
85+
86+
if (!mediaPipeDirectory.existsSync()) {
87+
io.stderr.writeln(
88+
'Could not find ${mediaPipeDirectory.absolute.path}. '
89+
'Folder does not exist.',
90+
);
91+
io.exit(1);
92+
}
93+
return mediaPipeDirectory;
94+
}
95+
96+
/// Looks for the sentinel file of this repository's root directory. This allows
97+
/// the `dart build` command to be run from various locations within the
98+
/// `google/mediapipe` repository and still correctly set paths for all of its
99+
/// operations.
100+
bool _isFlutterMediaPipeRoot(io.Directory dir) {
101+
return io.File(
102+
path.joinAll(
103+
[dir.absolute.path, sentinelFileName],
104+
),
105+
).existsSync();
106+
}
107+
108+
/// Builds any missing folders between the file and the root of the repository
109+
void ensureFolders(io.File file) {
110+
io.Directory parent = file.parent;
111+
List<io.Directory> dirsToCreate = [];
112+
while (!parent.existsSync()) {
113+
dirsToCreate.add(parent);
114+
parent = parent.parent;
115+
}
116+
for (io.Directory dir in dirsToCreate.reversed) {
117+
dir.createSync();
118+
}
119+
}
120+
}

0 commit comments

Comments
 (0)