From c7c9d8eea6c9fe44bceb6d78be35c5046f46bca8 Mon Sep 17 00:00:00 2001 From: David Iglesias Date: Tue, 19 Sep 2023 15:38:51 -0700 Subject: [PATCH] [web] Encode AssetManifest.bin as JSON and use that on the web. (#131382) This PR modifies the web build slightly to create an `AssetManifest.json`, that is a JSON(base64)-encoded version of the `AssetManifest.bin` file. _(This should enable all browsers to download the file without any interference, and all servers to serve it with the correct headers.)_ It also modifies Flutter's `AssetManifest` class so it loads and uses said file `if (kIsWeb)`. ### Issues * Fixes https://github.com/flutter/flutter/issues/124883 ### Tests * Unit tests added. * Some tests that run on the Web needed to be informed of the new filename, but their behavior didn't have to change (binary contents are the same across all platforms). * I've deployed a test app, so users affected by the BIN issue may take a look at the PR in action: * https://dit-tests.web.app --- .../lib/src/services/asset_bundle.dart | 4 +- .../lib/src/services/asset_manifest.dart | 28 +++++- .../test/painting/image_resolution_test.dart | 17 ++++ .../test/services/asset_bundle_test.dart | 16 ++++ .../test/services/asset_manifest_test.dart | 54 +++++++---- packages/flutter_tools/lib/src/asset.dart | 17 +++- .../test/general.shard/asset_bundle_test.dart | 96 +++++++++++++++++++ 7 files changed, 205 insertions(+), 27 deletions(-) diff --git a/packages/flutter/lib/src/services/asset_bundle.dart b/packages/flutter/lib/src/services/asset_bundle.dart index 238467b27db6..0c65ca323ca7 100644 --- a/packages/flutter/lib/src/services/asset_bundle.dart +++ b/packages/flutter/lib/src/services/asset_bundle.dart @@ -88,8 +88,8 @@ abstract class AssetBundle { Future loadString(String key, { bool cache = true }) async { final ByteData data = await load(key); // 50 KB of data should take 2-3 ms to parse on a Moto G4, and about 400 μs - // on a Pixel 4. - if (data.lengthInBytes < 50 * 1024) { + // on a Pixel 4. On the web we can't bail to isolates, though... + if (data.lengthInBytes < 50 * 1024 || kIsWeb) { return utf8.decode(Uint8List.sublistView(data)); } // For strings larger than 50 KB, run the computation in an isolate to diff --git a/packages/flutter/lib/src/services/asset_manifest.dart b/packages/flutter/lib/src/services/asset_manifest.dart index 3b27490098a8..c948067e4cb3 100644 --- a/packages/flutter/lib/src/services/asset_manifest.dart +++ b/packages/flutter/lib/src/services/asset_manifest.dart @@ -2,18 +2,22 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; + import 'package:flutter/foundation.dart'; import 'asset_bundle.dart'; import 'message_codecs.dart'; // We use .bin as the extension since it is well-known to represent -// data in some arbitrary binary format. Using a well-known extension here -// is important for web, because some web servers will not serve files with -// unrecognized file extensions by default. -// See https://github.com/flutter/flutter/issues/128456. +// data in some arbitrary binary format. const String _kAssetManifestFilename = 'AssetManifest.bin'; +// We use the same bin file for the web, but re-encoded as JSON(base64(bytes)) +// so it can be downloaded by even the dumbest of browsers. +// See https://github.com/flutter/flutter/issues/128456 +const String _kAssetManifestWebFilename = 'AssetManifest.bin.json'; + /// Contains details about available assets and their variants. /// See [Resolution-aware image assets](https://docs.flutter.dev/ui/assets-and-images#resolution-aware) /// to learn about asset variants and how to declare them. @@ -21,6 +25,22 @@ abstract class AssetManifest { /// Loads asset manifest data from an [AssetBundle] object and creates an /// [AssetManifest] object from that data. static Future loadFromAssetBundle(AssetBundle bundle) { + // The AssetManifest file contains binary data. + // + // On the web, the build process wraps this binary data in json+base64 so + // it can be transmitted over the network without special configuration + // (see #131382). + if (kIsWeb) { + // On the web, the AssetManifest is downloaded as a String, then + // json+base64-decoded to get to the binary data. + return bundle.loadStructuredData(_kAssetManifestWebFilename, (String jsonData) async { + // Decode the manifest JSON file to the underlying BIN, and convert to ByteData. + final ByteData message = ByteData.sublistView(base64.decode(json.decode(jsonData) as String)); + // Now we can keep operating as usual. + return _AssetManifestBin.fromStandardMessageCodecMessage(message); + }); + } + // On every other platform, the binary file contents are used directly. return bundle.loadStructuredBinaryData(_kAssetManifestFilename, _AssetManifestBin.fromStandardMessageCodecMessage); } diff --git a/packages/flutter/test/painting/image_resolution_test.dart b/packages/flutter/test/painting/image_resolution_test.dart index 2a784de9e076..e52f1b274b34 100644 --- a/packages/flutter/test/painting/image_resolution_test.dart +++ b/packages/flutter/test/painting/image_resolution_test.dart @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; @@ -22,6 +23,22 @@ class TestAssetBundle extends CachingAssetBundle { return const StandardMessageCodec().encodeMessage(_assetBundleMap)!; } + if (key == 'AssetManifest.bin.json') { + // Encode the manifest data that will be used by the app + final ByteData data = const StandardMessageCodec().encodeMessage(_assetBundleMap)!; + // Simulate the behavior of NetworkAssetBundle.load here, for web tests + return ByteData.sublistView( + utf8.encode( + json.encode( + base64.encode( + // Encode only the actual bytes of the buffer, and no more... + data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes) + ) + ) + ) + ); + } + loadCallCount[key] = loadCallCount[key] ?? 0 + 1; if (key == 'one') { return ByteData(1) diff --git a/packages/flutter/test/services/asset_bundle_test.dart b/packages/flutter/test/services/asset_bundle_test.dart index 2177935357c8..7641adaf1a8a 100644 --- a/packages/flutter/test/services/asset_bundle_test.dart +++ b/packages/flutter/test/services/asset_bundle_test.dart @@ -25,6 +25,22 @@ class TestAssetBundle extends CachingAssetBundle { .encodeMessage({'one': []})!; } + if (key == 'AssetManifest.bin.json') { + // Encode the manifest data that will be used by the app + final ByteData data = const StandardMessageCodec().encodeMessage( {'one': []})!; + // Simulate the behavior of NetworkAssetBundle.load here, for web tests + return ByteData.sublistView( + utf8.encode( + json.encode( + base64.encode( + // Encode only the actual bytes of the buffer, and no more... + data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes) + ) + ) + ) + ); + } + if (key == 'counter') { return ByteData.sublistView(utf8.encode(loadCallCount[key]!.toString())); } diff --git a/packages/flutter/test/services/asset_manifest_test.dart b/packages/flutter/test/services/asset_manifest_test.dart index c06ffd0126b6..108515f4330d 100644 --- a/packages/flutter/test/services/asset_manifest_test.dart +++ b/packages/flutter/test/services/asset_manifest_test.dart @@ -2,34 +2,52 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; + import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; class TestAssetBundle extends AssetBundle { + static const Map> _binManifestData = >{ + 'assets/foo.png': [ + { + 'asset': 'assets/foo.png', + }, + { + 'asset': 'assets/2x/foo.png', + 'dpr': 2.0 + }, + ], + 'assets/bar.png': [ + { + 'asset': 'assets/bar.png', + }, + ], + }; + @override Future load(String key) async { if (key == 'AssetManifest.bin') { - final Map> binManifestData = >{ - 'assets/foo.png': [ - { - 'asset': 'assets/foo.png', - }, - { - 'asset': 'assets/2x/foo.png', - 'dpr': 2.0 - }, - ], - 'assets/bar.png': [ - { - 'asset': 'assets/bar.png', - }, - ], - }; - - final ByteData data = const StandardMessageCodec().encodeMessage(binManifestData)!; + final ByteData data = const StandardMessageCodec().encodeMessage(_binManifestData)!; return data; } + if (key == 'AssetManifest.bin.json') { + // Encode the manifest data that will be used by the app + final ByteData data = const StandardMessageCodec().encodeMessage(_binManifestData)!; + // Simulate the behavior of NetworkAssetBundle.load here, for web tests + return ByteData.sublistView( + utf8.encode( + json.encode( + base64.encode( + // Encode only the actual bytes of the buffer, and no more... + data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes) + ) + ) + ) + ); + } + throw ArgumentError('Unexpected key'); } diff --git a/packages/flutter_tools/lib/src/asset.dart b/packages/flutter_tools/lib/src/asset.dart index 7c8f054517cd..7eda50bc1f7d 100644 --- a/packages/flutter_tools/lib/src/asset.dart +++ b/packages/flutter_tools/lib/src/asset.dart @@ -168,6 +168,7 @@ class ManifestAssetBundle implements AssetBundle { // We assume the main asset is designed for a device pixel ratio of 1.0. static const String _kAssetManifestJsonFilename = 'AssetManifest.json'; static const String _kAssetManifestBinFilename = 'AssetManifest.bin'; + static const String _kAssetManifestBinJsonFilename = 'AssetManifest.bin.json'; static const String _kNoticeFile = 'NOTICES'; // Comically, this can't be name with the more common .gz file extension @@ -233,8 +234,6 @@ class ManifestAssetBundle implements AssetBundle { // device. _lastBuildTimestamp = DateTime.now(); if (flutterManifest.isEmpty) { - entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}'); - entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular; entries[_kAssetManifestJsonFilename] = DevFSStringContent('{}'); entryKinds[_kAssetManifestJsonFilename] = AssetKind.regular; final ByteData emptyAssetManifest = @@ -242,6 +241,11 @@ class ManifestAssetBundle implements AssetBundle { entries[_kAssetManifestBinFilename] = DevFSByteContent(emptyAssetManifest.buffer.asUint8List(0, emptyAssetManifest.lengthInBytes)); entryKinds[_kAssetManifestBinFilename] = AssetKind.regular; + // Create .bin.json on web builds. + if (targetPlatform == TargetPlatform.web_javascript) { + entries[_kAssetManifestBinJsonFilename] = DevFSStringContent('""'); + entryKinds[_kAssetManifestBinJsonFilename] = AssetKind.regular; + } return 0; } @@ -437,8 +441,8 @@ class ManifestAssetBundle implements AssetBundle { final Map> assetManifest = _createAssetManifest(assetVariants, deferredComponentsAssetVariants); - final DevFSStringContent assetManifestJson = DevFSStringContent(json.encode(assetManifest)); final DevFSByteContent assetManifestBinary = _createAssetManifestBinary(assetManifest); + final DevFSStringContent assetManifestJson = DevFSStringContent(json.encode(assetManifest)); final DevFSStringContent fontManifest = DevFSStringContent(json.encode(fonts)); final LicenseResult licenseResult = _licenseCollector.obtainLicenses(packageConfig, additionalLicenseFiles); if (licenseResult.errorMessages.isNotEmpty) { @@ -464,6 +468,13 @@ class ManifestAssetBundle implements AssetBundle { _setIfChanged(_kAssetManifestJsonFilename, assetManifestJson, AssetKind.regular); _setIfChanged(_kAssetManifestBinFilename, assetManifestBinary, AssetKind.regular); + // Create .bin.json on web builds. + if (targetPlatform == TargetPlatform.web_javascript) { + final DevFSStringContent assetManifestBinaryJson = DevFSStringContent(json.encode( + base64.encode(assetManifestBinary.bytes) + )); + _setIfChanged(_kAssetManifestBinJsonFilename, assetManifestBinaryJson, AssetKind.regular); + } _setIfChanged(kFontManifestJson, fontManifest, AssetKind.regular); _setLicenseIfChanged(licenseResult.combinedLicenses, targetPlatform); return 0; diff --git a/packages/flutter_tools/test/general.shard/asset_bundle_test.dart b/packages/flutter_tools/test/general.shard/asset_bundle_test.dart index 11837c7fb257..4b798f61ec66 100644 --- a/packages/flutter_tools/test/general.shard/asset_bundle_test.dart +++ b/packages/flutter_tools/test/general.shard/asset_bundle_test.dart @@ -325,6 +325,102 @@ flutter: }); }); + group('AssetBundle.build (web builds)', () { + late FileSystem testFileSystem; + + setUp(() async { + testFileSystem = MemoryFileSystem( + style: globals.platform.isWindows + ? FileSystemStyle.windows + : FileSystemStyle.posix, + ); + testFileSystem.currentDirectory = testFileSystem.systemTempDirectory.createTempSync('flutter_asset_bundle_test.'); + }); + + testUsingContext('empty pubspec', () async { + globals.fs.file('pubspec.yaml') + ..createSync() + ..writeAsStringSync(''); + + final AssetBundle bundle = AssetBundleFactory.instance.createBundle(); + await bundle.build(packagesPath: '.packages', targetPlatform: TargetPlatform.web_javascript); + + expect(bundle.entries.keys, + unorderedEquals([ + 'AssetManifest.json', + 'AssetManifest.bin', + 'AssetManifest.bin.json', + ]) + ); + expect( + utf8.decode(await bundle.entries['AssetManifest.json']!.contentsAsBytes()), + '{}', + ); + expect( + utf8.decode(await bundle.entries['AssetManifest.bin.json']!.contentsAsBytes()), + '""', + ); + }, overrides: { + FileSystem: () => testFileSystem, + ProcessManager: () => FakeProcessManager.any(), + }); + + testUsingContext('pubspec contains an asset', () async { + globals.fs.file('.packages').createSync(); + globals.fs.file('pubspec.yaml').writeAsStringSync(r''' +name: test +dependencies: + flutter: + sdk: flutter +flutter: + assets: + - assets/bar/lizard.png +'''); + globals.fs.file( + globals.fs.path.joinAll(['assets', 'bar', 'lizard.png']) + ).createSync(recursive: true); + + final AssetBundle bundle = AssetBundleFactory.instance.createBundle(); + await bundle.build(packagesPath: '.packages', targetPlatform: TargetPlatform.web_javascript); + + expect(bundle.entries.keys, + unorderedEquals([ + 'AssetManifest.json', + 'AssetManifest.bin', + 'AssetManifest.bin.json', + 'FontManifest.json', + 'NOTICES', // not .Z + 'assets/bar/lizard.png', + ]) + ); + final Map manifestJson = json.decode( + utf8.decode( + await bundle.entries['AssetManifest.json']!.contentsAsBytes() + ) + ) as Map; + expect(manifestJson, isNotEmpty); + expect(manifestJson['assets/bar/lizard.png'], isNotNull); + + final Uint8List manifestBinJsonBytes = base64.decode( + json.decode( + utf8.decode( + await bundle.entries['AssetManifest.bin.json']!.contentsAsBytes() + ) + ) as String + ); + + final Uint8List manifestBinBytes = Uint8List.fromList( + await bundle.entries['AssetManifest.bin']!.contentsAsBytes() + ); + + expect(manifestBinJsonBytes, equals(manifestBinBytes), + reason: 'JSON-encoded binary content should be identical to BIN file.'); + }, overrides: { + FileSystem: () => testFileSystem, + ProcessManager: () => FakeProcessManager.any(), + }); + }); + testUsingContext('Failed directory delete shows message', () async { final FileExceptionHandler handler = FileExceptionHandler(); final FileSystem fileSystem = MemoryFileSystem.test(opHandle: handler.opHandle);