From dcc318b88a9aec7f92d5839e0c94f00e7e0e798b Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Mon, 8 Feb 2016 10:07:52 -0800 Subject: [PATCH] track build outputs in a file for subsequent builds --- .gitignore | 3 ++ lib/src/asset/cache.dart | 6 ++++ lib/src/asset/file_based.dart | 16 +++++++--- lib/src/asset/writer.dart | 3 ++ lib/src/generate/build.dart | 38 +++++++++++++++++++---- lib/src/transformer/transformer.dart | 5 ++++ test/asset/cache_test.dart | 7 +++++ test/asset/file_based_test.dart | 22 ++++++++++---- test/common/in_memory_writer.dart | 4 +++ test/common/stub_writer.dart | 2 ++ test/generate/build_test.dart | 45 +++++++++++++++++++++++++--- 11 files changed, 133 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index d8fd35055..76e53c12b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ build/ .pub/ pubspec.lock +# Files generated by build runs +.build + # Generated by tool/create_all_test.dart tool/test_all.dart diff --git a/lib/src/asset/cache.dart b/lib/src/asset/cache.dart index ef7663d19..a92b4c38b 100644 --- a/lib/src/asset/cache.dart +++ b/lib/src/asset/cache.dart @@ -97,4 +97,10 @@ class CachedAssetWriter extends AssetWriter { _cache.put(asset); return _writer.writeAsString(asset, encoding: encoding); } + + @override + Future delete(AssetId id) { + _cache.remove(id); + return _writer.delete(id); + } } diff --git a/lib/src/asset/file_based.dart b/lib/src/asset/file_based.dart index 8839d55d6..de53df562 100644 --- a/lib/src/asset/file_based.dart +++ b/lib/src/asset/file_based.dart @@ -56,10 +56,8 @@ class FileBasedAssetReader implements AssetReader { var packageNode = packageGraph[inputSet.package]; var packagePath = packageNode.location.toFilePath(); for (var glob in inputSet.globs) { - var fileStream = glob - .list(followLinks: false, root: packagePath) - .where((e) => - e is File && !ignoredDirs.contains(path.split(e.path)[1])); + var fileStream = glob.list(followLinks: false, root: packagePath).where( + (e) => e is File && !ignoredDirs.contains(path.split(e.path)[1])); await for (var entity in fileStream) { var id = _fileToAssetId(entity, packageNode); if (!seenAssets.add(id)) continue; @@ -98,6 +96,16 @@ class FileBasedAssetWriter implements AssetWriter { await file.create(recursive: true); await file.writeAsString(asset.stringContents, encoding: encoding); } + + @override + delete(AssetId id) async { + assert(id.package == packageGraph.root.name); + + var file = _fileFor(id, packageGraph); + if (await file.exists()) { + await file.delete(); + } + } } /// Returns a [File] for [id] given [packageGraph]. diff --git a/lib/src/asset/writer.dart b/lib/src/asset/writer.dart index 733363085..9e0b65ffb 100644 --- a/lib/src/asset/writer.dart +++ b/lib/src/asset/writer.dart @@ -5,7 +5,10 @@ import 'dart:async'; import 'dart:convert'; import 'asset.dart'; +import 'id.dart'; abstract class AssetWriter { Future writeAsString(Asset asset, {Encoding encoding: UTF8}); + + Future delete(AssetId id); } diff --git a/lib/src/generate/build.dart b/lib/src/generate/build.dart index 54d821af5..a408b5ff1 100644 --- a/lib/src/generate/build.dart +++ b/lib/src/generate/build.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; @@ -49,17 +50,43 @@ Future build(List> phaseGroups, new CachedAssetReader(cache, new FileBasedAssetReader(packageGraph)); writer ??= new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph)); - var result = runZoned(() { - return _runPhases(phaseGroups); - }, onError: (e, s) { - return new BuildResult(BuildStatus.Failure, BuildType.Full, [], - exception: e, stackTrace: s); + + /// Asset containing previous build outputs. + var buildOuputsId = + new AssetId(packageGraph.root.name, '.build/build_outputs.json'); + + /// Run the build in a zone. + var result = await runZoned(() async { + try { + /// Read in previous build_outputs file, and delete them all. + if (await _reader.hasInput(buildOuputsId)) { + var previousOutputs = + JSON.decode(await _reader.readAsString(buildOuputsId)); + await writer.delete(buildOuputsId); + await Future.wait(previousOutputs.map((output) { + return writer.delete(new AssetId.deserialize(output)); + })); + } + + /// Run a fresh build. + return _runPhases(phaseGroups); + } catch(e, s) { + return new BuildResult(BuildStatus.Failure, BuildType.Full, [], + exception: e, stackTrace: s); + } }, zoneValues: { _assetReaderKey: reader, _assetWriterKey: writer, _packageGraphKey: packageGraph, }); + await logListener.cancel(); + + // Write out the new build_outputs file. + var buildOutputsAsset = new Asset(buildOuputsId, + JSON.encode(result.outputs.map((output) => output.id.serialize()).toList())); + await writer.writeAsString(buildOutputsAsset); + return result; } @@ -91,6 +118,7 @@ Future _runPhases(List> phaseGroups) async { } } } + /// Once the group is done, add all outputs so they can be used in the next /// phase. for (var output in groupOutputs) { diff --git a/lib/src/transformer/transformer.dart b/lib/src/transformer/transformer.dart index a566a675f..89017d816 100644 --- a/lib/src/transformer/transformer.dart +++ b/lib/src/transformer/transformer.dart @@ -99,6 +99,7 @@ class _TransformAssetReader implements AssetReader { transform.readInputAsString(toBarbackAssetId(id), encoding: encoding); @override + /// No way to implement this, but luckily its not necessary. Stream listAssetIds(_) => throw new UnimplementedError(); } @@ -114,6 +115,10 @@ class _TransformAssetWriter implements AssetWriter { transform.addOutput(toBarbackAsset(asset)); return new Future.value(null); } + + @override + Future delete(build.AssetId id) => + throw new UnsupportedError('_TransformAssetWriter can\'t delete files.'); } /// All the expected outputs for [id] given [builders]. diff --git a/test/asset/cache_test.dart b/test/asset/cache_test.dart index f24da3df1..d112de76b 100644 --- a/test/asset/cache_test.dart +++ b/test/asset/cache_test.dart @@ -170,5 +170,12 @@ main() { await writer.writeAsString(a); expect(() => writer.writeAsString(a), throwsArgumentError); }); + + test('delete deletes from cache and writer', () async { + await writer.writeAsString(a); + await writer.delete(a.id); + expect(cache.get(a.id), isNull); + expect(childWriterAssets[a.id], isNull); + }); }); } diff --git a/test/asset/file_based_test.dart b/test/asset/file_based_test.dart index 85c30129f..d99f38915 100644 --- a/test/asset/file_based_test.dart +++ b/test/asset/file_based_test.dart @@ -69,15 +69,15 @@ main() { test('can list files based on simple InputSets', () async { var inputSets = [ - new InputSet('basic_pkg'), - new InputSet('a'), + new InputSet('basic_pkg', filePatterns: ['{lib,web}/**']), + new InputSet('a', filePatterns: ['lib/**']), ]; expect(await reader.listAssetIds(inputSets).toList(), unorderedEquals([ makeAssetId('basic_pkg|lib/hello.txt'), makeAssetId('basic_pkg|web/hello.txt'), makeAssetId('a|lib/a.txt'), ])); - }, skip: 'Fails for multiple reasons.'); + }); test('can list files based on InputSets with globs', () async { var inputSets = [ @@ -94,14 +94,16 @@ main() { group('FileBasedAssetWriter', () { final writer = new FileBasedAssetWriter(packageGraph); - test('can output files in the application package', () async { + test('can output and delete files in the application package', () async { var asset = makeAsset('basic_pkg|test_file.txt', 'test'); await writer.writeAsString(asset); var id = asset.id; var file = new File('test/fixtures/${id.package}/${id.path}'); expect(await file.exists(), isTrue); expect(await file.readAsString(), 'test'); - await file.delete(); + + await writer.delete(asset.id); + expect(await file.exists(), isFalse); }); test('can\'t output files in package dependencies', () async { @@ -113,5 +115,15 @@ main() { var asset = makeAsset('foo|bar.txt'); expect(writer.writeAsString(asset), throwsA(invalidOutputException)); }); + + test('can\'t delete files in package dependencies', () async { + var id = makeAssetId('a|test.txt'); + expect(writer.delete(id), throws); + }); + + test('can\'t delete files in arbitrary dependencies', () async { + var id = makeAssetId('foo|test.txt'); + expect(writer.delete(id), throws); + }); }); } diff --git a/test/common/in_memory_writer.dart b/test/common/in_memory_writer.dart index a43f2e06b..64f17f14a 100644 --- a/test/common/in_memory_writer.dart +++ b/test/common/in_memory_writer.dart @@ -14,4 +14,8 @@ class InMemoryAssetWriter implements AssetWriter { Future writeAsString(Asset asset, {Encoding encoding: UTF8}) async { assets[asset.id] = asset.stringContents; } + + Future delete(AssetId id) async { + assets.remove(id); + } } diff --git a/test/common/stub_writer.dart b/test/common/stub_writer.dart index 64d747665..cb0d755be 100644 --- a/test/common/stub_writer.dart +++ b/test/common/stub_writer.dart @@ -9,4 +9,6 @@ import 'package:build/build.dart'; class StubAssetWriter implements AssetWriter { Future writeAsString(Asset asset, {Encoding encoding: UTF8}) => new Future.value(null); + + Future delete(AssetId id) => new Future.value(null); } diff --git a/test/generate/build_test.dart b/test/generate/build_test.dart index b9f5d256a..042c78a1e 100644 --- a/test/generate/build_test.dart +++ b/test/generate/build_test.dart @@ -1,6 +1,8 @@ // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; + import 'package:test/test.dart'; import 'package:build/build.dart'; @@ -172,17 +174,52 @@ main() { 'the BuildStep level.'); }); + test('tracks previous outputs in a build_outputs.json file', () async { + var phases = [ + [ + new Phase([new CopyBuilder()], [new InputSet('a')]), + ] + ]; + final writer = new InMemoryAssetWriter(); + await testPhases(phases, {'a|web/a.txt': 'a', 'a|lib/b.txt': 'b'}, + outputs: {'a|web/a.txt.copy': 'a', 'a|lib/b.txt.copy': 'b'}, + writer: writer); + + var outputId = makeAssetId('a|.build/build_outputs.json'); + expect(writer.assets, contains(outputId)); + var outputs = JSON.decode(writer.assets[outputId]); + expect( + outputs, + unorderedEquals([ + ['a', 'web/a.txt.copy'], + ['a', 'lib/b.txt.copy'], + ])); + }); + test('outputs from previous full builds shouldn\'t be inputs to later ones', - () {}, - skip: 'Unimplemented: https://github.com/dart-lang/build/issues/34'); + () async { + var phases = [ + [ + new Phase([new CopyBuilder()], [new InputSet('a')]), + ] + ]; + final writer = new InMemoryAssetWriter(); + var inputs = {'a|web/a.txt': 'a', 'a|lib/b.txt': 'b'}; + var outputs = {'a|web/a.txt.copy': 'a', 'a|lib/b.txt.copy': 'b'}; + // First run, nothing special. + await testPhases(phases, inputs, outputs: outputs, writer: writer); + // Second run, should have no extra outputs. + await testPhases(phases, inputs, outputs: outputs, writer: writer); + }); } testPhases(List> phases, Map inputs, {Map outputs, PackageGraph packageGraph, BuildStatus status: BuildStatus.Success, - exceptionMatcher}) async { - final writer = new InMemoryAssetWriter(); + exceptionMatcher, + InMemoryAssetWriter writer}) async { + writer ??= new InMemoryAssetWriter(); final actualAssets = writer.assets; final reader = new InMemoryAssetReader(actualAssets); inputs.forEach((serializedId, contents) {