diff --git a/build_runner/lib/src/asset/reader.dart b/build_runner/lib/src/asset/reader.dart index 7f6d783a5..a7e196614 100644 --- a/build_runner/lib/src/asset/reader.dart +++ b/build_runner/lib/src/asset/reader.dart @@ -57,7 +57,7 @@ class SingleStepReader implements AssetReader { SingleStepReader(this._delegate, this._assetGraph, this._phaseNumber, this._primaryPackage, this._runPhaseForInput); - Iterable get assetsRead => _assetsRead; + Set get assetsRead => _assetsRead; /// The [Glob]s which have been searched with [findAssets]. /// diff --git a/build_runner/lib/src/asset_graph/graph.dart b/build_runner/lib/src/asset_graph/graph.dart index af087fac5..c4e9d3c26 100644 --- a/build_runner/lib/src/asset_graph/graph.dart +++ b/build_runner/lib/src/asset_graph/graph.dart @@ -109,6 +109,12 @@ class AssetGraph { for (var output in node.primaryOutputs) { _remove(output, removedIds: removedIds); } + for (var output in node.outputs) { + var generatedNode = get(output) as GeneratedAssetNode; + if (generatedNode != null) { + generatedNode.inputs.remove(id); + } + } _nodesByPackage[id.package].remove(id.path); return removedIds; } @@ -145,12 +151,10 @@ class AssetGraph { // Builds up `idsToDelete` and `idsToRemove` by recursively invalidating // the outputs of `id`. - void clearNodeAndDeps(AssetId id, ChangeType rootChangeType, - {bool rootIsSource}) { + void clearNodeAndDeps(AssetId id, ChangeType rootChangeType) { var node = this.get(id); if (node == null) return; if (!invalidatedIds.add(id)) return; - rootIsSource ??= node is SourceAssetNode; if (node is GeneratedAssetNode) { idsToDelete.add(id); @@ -161,7 +165,7 @@ class AssetGraph { // Update all outputs of this asset as well. for (var output in node.outputs) { - clearNodeAndDeps(output, rootChangeType, rootIsSource: rootIsSource); + clearNodeAndDeps(output, rootChangeType); } } diff --git a/build_runner/lib/src/asset_graph/node.dart b/build_runner/lib/src/asset_graph/node.dart index bc78b733e..64179802b 100644 --- a/build_runner/lib/src/asset_graph/node.dart +++ b/build_runner/lib/src/asset_graph/node.dart @@ -72,10 +72,15 @@ class GeneratedAssetNode extends AssetNode { /// Any new or deleted files matching this glob should invalidate this node. Set globs; + /// All the inputs that were read when generating this asset, or deciding not + /// to generate it. + final Set inputs; + GeneratedAssetNode(this.phaseNumber, this.primaryInput, this.needsUpdate, this.wasOutput, AssetId id, - {Digest lastKnownDigest, Set globs}) + {Digest lastKnownDigest, Set globs, Iterable inputs}) : this.globs = globs ?? new Set(), + this.inputs = inputs?.toSet() ?? new Set(), super(id, lastKnownDigest: lastKnownDigest); @override diff --git a/build_runner/lib/src/asset_graph/serialization.dart b/build_runner/lib/src/asset_graph/serialization.dart index ee8ee8853..340b38932 100644 --- a/build_runner/lib/src/asset_graph/serialization.dart +++ b/build_runner/lib/src/asset_graph/serialization.dart @@ -32,10 +32,23 @@ class _AssetGraphDeserializer { new AssetId(descriptor[1] as String, descriptor[2] as String); } + // Read in all the nodes and their outputs. + // + // Note that this does not read in the inputs of generated nodes. for (var serializedItem in _serializedGraph['nodes']) { graph._add(_deserializeAssetNode(serializedItem as List)); } + // Update the inputs of all generated nodes based on the outputs of the + // current nodes. + for (var node in graph.allNodes) { + for (var output in node.outputs) { + var generatedNode = graph.get(output) as GeneratedAssetNode; + assert(generatedNode != null, 'Asset Graph is missing $output'); + generatedNode.inputs.add(node.id); + } + } + return graph; } diff --git a/build_runner/lib/src/generate/build_impl.dart b/build_runner/lib/src/generate/build_impl.dart index 363bfc496..041bc9698 100644 --- a/build_runner/lib/src/generate/build_impl.dart +++ b/build_runner/lib/src/generate/build_impl.dart @@ -411,12 +411,38 @@ class BuildImpl { SingleStepReader reader, AssetWriterSpy writer) async { // Reset the state for each output, setting `wasOutput` to false for now // (will set to true in the next loop for written assets). + // + // Also updates the `inputs` set for each output, and the `outputs` sets for + // all inputs. for (var output in declaredOutputs) { - (_assetGraph.get(output) as GeneratedAssetNode) + var node = _assetGraph.get(output) as GeneratedAssetNode; + node ..needsUpdate = false ..wasOutput = false ..lastKnownDigest = null ..globs = reader.globsRan.toSet(); + + // Update dependencies that don't exist any more. + var removedInputs = node.inputs.difference(reader.assetsRead); + node.inputs.removeAll(removedInputs); + for (var input in removedInputs) { + // TODO: special type of dependency here? This means the primary input + // was never actually read. + if (input == node.primaryInput) continue; + + var inputNode = _assetGraph.get(input); + assert(inputNode != null, 'Asset Graph is missing $input'); + inputNode.outputs.remove(output); + } + + // Add the new dependencies. + var newInputs = reader.assetsRead.difference(node.inputs); + node.inputs.addAll(newInputs); + for (var input in newInputs) { + var inputNode = _assetGraph.get(input); + assert(inputNode != null, 'Asset Graph is missing $input'); + inputNode.outputs.add(output); + } } // Mark the actual outputs as output. @@ -425,15 +451,6 @@ class BuildImpl { ..wasOutput = true ..lastKnownDigest = await _reader.digest(output); })); - - // Update the asset graph based on the dependencies discovered. - for (var dependency in reader.assetsRead) { - var dependencyNode = _assetGraph.get(dependency); - assert(dependencyNode != null, 'Asset Graph is missing $dependency'); - // We care about all builderOutputs, not just real outputs. Updates - // to dependencies may cause a file to be output which wasn't before. - dependencyNode.outputs.addAll(declaredOutputs); - } } Future _delete(AssetId id) { diff --git a/build_runner/test/asset_graph/graph_test.dart b/build_runner/test/asset_graph/graph_test.dart index 11ae50c2e..59df00f4f 100644 --- a/build_runner/test/asset_graph/graph_test.dart +++ b/build_runner/test/asset_graph/graph_test.dart @@ -88,6 +88,8 @@ void main() { var syntheticNode = new SyntheticAssetNode(makeAssetId()); syntheticNode.outputs.add(generatedNode.id); + generatedNode.inputs.addAll([node.id, syntheticNode.id]); + graph.add(syntheticNode); graph.add(generatedNode); } diff --git a/build_runner/test/common/matchers.dart b/build_runner/test/common/matchers.dart index bd1175c69..6566679c9 100644 --- a/build_runner/test/common/matchers.dart +++ b/build_runner/test/common/matchers.dart @@ -84,6 +84,14 @@ class _AssetGraphMatcher extends Matcher { ]; matches = false; } + if (!unorderedEquals(node.inputs) + .matches(expectedNode.inputs, null)) { + matchState['Inputs of ${node.id}'] = [ + node.inputs, + expectedNode.inputs + ]; + matches = false; + } } else { matchState['Type of ${node.id}'] = [ node.runtimeType, diff --git a/build_runner/test/generate/build_test.dart b/build_runner/test/generate/build_test.dart index 786468a86..04ff4a3a9 100644 --- a/build_runner/test/generate/build_test.dart +++ b/build_runner/test/generate/build_test.dart @@ -418,13 +418,15 @@ void main() { var expectedGraph = await AssetGraph.build([], new Set(), 'a', null); var aCopyNode = new GeneratedAssetNode(null, makeAssetId('a|web/a.txt'), false, true, makeAssetId('a|web/a.txt.copy'), - lastKnownDigest: computeDigest('a')); + lastKnownDigest: computeDigest('a'), + inputs: [makeAssetId('a|web/a.txt')]); expectedGraph.add(aCopyNode); expectedGraph .add(makeAssetNode('a|web/a.txt', [aCopyNode.id], computeDigest('a'))); var bCopyNode = new GeneratedAssetNode(null, makeAssetId('a|lib/b.txt'), false, true, makeAssetId('a|lib/b.txt.copy'), - lastKnownDigest: computeDigest('b')); + lastKnownDigest: computeDigest('b'), + inputs: [makeAssetId('a|lib/b.txt')]); expectedGraph.add(bCopyNode); expectedGraph .add(makeAssetNode('a|lib/b.txt', [bCopyNode.id], computeDigest('b'))); @@ -473,94 +475,38 @@ void main() { group('incremental builds with cached graph', () { test('one new asset, one modified asset, one unchanged asset', () async { - var graph = await AssetGraph.build([], new Set(), 'a', null); - var bId = makeAssetId('a|lib/b.txt'); - var bCopyNode = new GeneratedAssetNode( - 0, bId, false, true, makeAssetId('a|lib/b.txt.copy'), - lastKnownDigest: computeDigest('b')); - graph.add(bCopyNode); - var bNode = - makeAssetNode('a|lib/b.txt', [bCopyNode.id], computeDigest('b')); - graph.add(bNode); + var buildActions = [copyABuildAction]; + // Initial build. var writer = new InMemoryRunnerAssetWriter(); - await writer.writeAsString(makeAssetId('a|lib/b.txt'), 'b'); - await testActions([ - copyABuildAction - ], { - 'a|web/a.txt': 'a', - 'a|lib/b.txt.copy': 'b', - 'a|lib/c.txt': 'c', - 'a|$assetGraphPath': JSON.encode(graph.serialize()), - }, outputs: { - 'a|web/a.txt.copy': 'a', - 'a|lib/c.txt.copy': 'c', - }, writer: writer); - }); - - test('invalidates generated assets based on graph age', () async { - var buildActions = [ - copyABuildAction, - new BuildAction(new CopyBuilder(extension: 'clone'), 'a', - inputs: ['**/*.txt.copy']) - ]; - - var graph = await AssetGraph.build([], new Set(), 'a', null); - - var aCloneNode = new GeneratedAssetNode( - 1, - makeAssetId('a|lib/a.txt.copy'), - false, - true, - makeAssetId('a|lib/a.txt.copy.clone'), - lastKnownDigest: computeDigest('a')); - graph.add(aCloneNode); - var aCopyNode = new GeneratedAssetNode(0, makeAssetId('a|lib/a.txt'), - false, true, makeAssetId('a|lib/a.txt.copy'), - lastKnownDigest: computeDigest('a')) - ..primaryOutputs.add(aCloneNode.id) - ..outputs.add(aCloneNode.id); - graph.add(aCopyNode); - var aNode = - makeAssetNode('a|lib/a.txt', [aCopyNode.id], computeDigest('a')) - ..primaryOutputs.add(aCopyNode.id) - ..outputs.add(aCopyNode.id); - graph.add(aNode); - - var bCloneNode = new GeneratedAssetNode( - 1, - makeAssetId('a|lib/b.txt.copy'), - false, - true, - makeAssetId('a|lib/b.txt.copy.clone'), - lastKnownDigest: computeDigest('b')); - graph.add(bCloneNode); - var bCopyNode = new GeneratedAssetNode(0, makeAssetId('a|lib/b.txt'), - false, true, makeAssetId('a|lib/b.txt.copy'), - lastKnownDigest: computeDigest('b')) - ..primaryOutputs.add(bCloneNode.id) - ..outputs.add(bCloneNode.id); - graph.add(bCopyNode); - var bNode = - makeAssetNode('a|lib/b.txt', [bCopyNode.id], computeDigest('b')) - ..primaryOutputs.add(bCopyNode.id); - graph.add(bNode); + await testActions( + buildActions, + { + 'a|web/a.txt': 'a', + 'a|lib/b.txt': 'b', + }, + outputs: { + 'a|web/a.txt.copy': 'a', + 'a|lib/b.txt.copy': 'b', + }, + writer: writer); - var writer = new InMemoryRunnerAssetWriter(); - await writer.writeAsString(makeAssetId('a|lib/b.txt'), 'b'); + // Followup build with modified inputs. + var serializedGraph = writer.assets[makeAssetId('a|$assetGraphPath')]; + writer.assets.clear(); await testActions( buildActions, { - 'a|lib/a.txt': 'b', - 'a|lib/a.txt.copy': 'a', - 'a|lib/a.txt.copy.clone': 'a', + 'a|web/a.txt': 'a2', + 'a|web/a.txt.copy': 'a', + 'a|lib/b.txt': 'b', 'a|lib/b.txt.copy': 'b', - 'a|lib/b.txt.copy.clone': 'b', - 'a|$assetGraphPath': JSON.encode(graph.serialize()), + 'a|lib/c.txt': 'c', + 'a|$assetGraphPath': serializedGraph, }, outputs: { - 'a|lib/a.txt.copy': 'b', - 'a|lib/a.txt.copy.clone': 'b', + 'a|web/a.txt.copy': 'a2', + 'a|lib/c.txt.copy': 'c', }, writer: writer); }); @@ -572,34 +518,28 @@ void main() { inputs: ['**/*.txt.copy']) ]; - var graph = await AssetGraph.build([], new Set(), 'a', null); - var aCloneNode = new GeneratedAssetNode( - 0, - makeAssetId('a|lib/a.txt.copy'), - false, - true, - makeAssetId('a|lib/a.txt.copy.clone'), - lastKnownDigest: computeDigest('a')); - graph.add(aCloneNode); - var aCopyNode = new GeneratedAssetNode(0, makeAssetId('a|lib/a.txt'), - false, true, makeAssetId('a|lib/a.txt.copy'), - lastKnownDigest: computeDigest('a')) - ..primaryOutputs.add(aCloneNode.id) - ..outputs.add(aCloneNode.id); - graph.add(aCopyNode); - var aNode = - makeAssetNode('a|lib/a.txt', [aCopyNode.id], computeDigest('a')) - ..primaryOutputs.add(aCopyNode.id) - ..outputs.add(aCopyNode.id); - graph.add(aNode); - + // Initial build. var writer = new InMemoryRunnerAssetWriter(); + await testActions( + buildActions, + { + 'a|lib/a.txt': 'a', + }, + outputs: { + 'a|lib/a.txt.copy': 'a', + 'a|lib/a.txt.copy.clone': 'a', + }, + writer: writer); + + // Followup build with deleted input + cached graph. + var serializedGraph = writer.assets[makeAssetId('a|$assetGraphPath')]; + writer.assets.clear(); await testActions( buildActions, { 'a|lib/a.txt.copy': 'a', 'a|lib/a.txt.copy.clone': 'a', - 'a|$assetGraphPath': JSON.encode(graph.serialize()), + 'a|$assetGraphPath': serializedGraph, }, outputs: {}, writer: writer); @@ -608,69 +548,107 @@ void main() { var serialized = JSON.decode( UTF8.decode(writer.assets[makeAssetId('a|$assetGraphPath')])) as Map; var newGraph = new AssetGraph.deserialize(serialized); - expect(newGraph.contains(aNode.id), isFalse); - expect(newGraph.contains(aCopyNode.id), isFalse); - expect(newGraph.contains(aCloneNode.id), isFalse); - expect(writer.assets.containsKey(aNode.id), isFalse); - expect(writer.assets.containsKey(aCopyNode.id), isFalse); - expect(writer.assets.containsKey(aCloneNode.id), isFalse); + var aNodeId = makeAssetId('a|lib/a.txt'); + var aCopyNodeId = makeAssetId('a|lib/a.txt.copy'); + var aCloneNodeId = makeAssetId('a|lib/a.txt.copy.clone'); + expect(newGraph.contains(aNodeId), isFalse); + expect(newGraph.contains(aCopyNodeId), isFalse); + expect(newGraph.contains(aCloneNodeId), isFalse); + expect(writer.assets.containsKey(aNodeId), isFalse); + expect(writer.assets.containsKey(aCopyNodeId), isFalse); + expect(writer.assets.containsKey(aCloneNodeId), isFalse); }); test('no outputs if no changed sources', () async { - var graph = await AssetGraph.build([], new Set(), 'a', null); - var aId = makeAssetId('a|web/a.txt'); - var aCopyNode = new GeneratedAssetNode( - 0, aId, false, true, makeAssetId('a|web/a.txt.copy'), - lastKnownDigest: computeDigest('a')); - graph.add(aCopyNode); - var aNode = - makeAssetNode('a|web/a.txt', [aCopyNode.id], computeDigest('a')); - graph.add(aNode); + var buildActions = [copyABuildAction]; + // Initial build. var writer = new InMemoryRunnerAssetWriter(); + await testActions(buildActions, {'a|web/a.txt': 'a'}, + outputs: {'a|web/a.txt.copy': 'a'}, writer: writer); - await writer.writeAsString(makeAssetId('a|web/a.txt'), ''); - await testActions([ - copyABuildAction - ], { + // Followup build with same sources + cached graph. + var serializedGraph = writer.assets[makeAssetId('a|$assetGraphPath')]; + await testActions(buildActions, { 'a|web/a.txt': 'a', 'a|web/a.txt.copy': 'a', - 'a|$assetGraphPath': JSON.encode(graph.serialize()), - }, outputs: {}, writer: writer); + 'a|$assetGraphPath': serializedGraph, + }, outputs: {}); }); test('no outputs if no changed sources using `writeToCache`', () async { - var graph = await AssetGraph.build([], new Set(), 'a', null); - var aId = makeAssetId('a|web/a.txt'); - var aCopyNode = new GeneratedAssetNode( - 0, aId, false, true, makeAssetId('a|web/a.txt.copy'), - lastKnownDigest: computeDigest('a')); - graph.add(aCopyNode); - var aNode = - makeAssetNode('a|web/a.txt', [aCopyNode.id], computeDigest('a')); - graph.add(aNode); + var buildActions = [copyABuildAction]; + // Initial build. var writer = new InMemoryRunnerAssetWriter(); + await testActions(buildActions, {'a|web/a.txt': 'a'}, + // Note that `testActions` converts generated cache dir paths to the + // original ones for matching. + outputs: {'a|web/a.txt.copy': 'a'}, + writer: writer, + writeToCache: true); - await writer.writeAsString(makeAssetId('a|web/a.txt'), ''); - await writer.writeAsString( - makeAssetId('a|.dart_tool/build/generated/a/web/a.txt'), ''); + // Followup build with same sources + cached graph. + var serializedGraph = writer.assets[makeAssetId('a|$assetGraphPath')]; + await testActions( + buildActions, + { + 'a|web/a.txt': 'a', + 'a|web/a.txt.copy': 'a', + 'a|$assetGraphPath': serializedGraph, + }, + outputs: {}, + writeToCache: true); + }); - var packageA = new PackageNode( - 'a', '0.1.0', PackageDependencyType.path, 'a/', - includes: ['**']); - var packageGraph = new PackageGraph.fromRoot(packageA); + test('inputs/outputs are updated if they change', () async { + // Initial build. + var writer = new InMemoryRunnerAssetWriter(); await testActions([ - copyABuildAction + new BuildAction( + new CopyBuilder(copyFromAsset: makeAssetId('a|lib/b.txt')), 'a', + inputs: ['lib/a.txt']) ], { - 'a|web/a.txt': 'a', - 'a|.dart_tool/build/generated/a/web/a.txt.copy': 'a', - 'a|$assetGraphPath': JSON.encode(graph.serialize()), - }, - outputs: {}, - writer: writer, - writeToCache: true, - packageGraph: packageGraph); + 'a|lib/a.txt': 'a', + 'a|lib/b.txt': 'b', + 'a|lib/c.txt': 'c', + }, outputs: { + 'a|lib/a.txt.copy': 'b', + }, writer: writer); + + // Followup build with same sources + cached graph, but configure the + // builder to read a different file. + var serializedGraph = writer.assets[makeAssetId('a|$assetGraphPath')]; + writer.assets.clear(); + + await testActions([ + new BuildAction( + new CopyBuilder(copyFromAsset: makeAssetId('a|lib/c.txt')), 'a', + inputs: ['lib/a.txt']) + ], { + 'a|lib/a.txt': 'a', + 'a|lib/a.txt.copy': 'b', + // Hack to get the file to rebuild, we are being bad by changing the + // builder but pretending its the same. + 'a|lib/b.txt': 'b2', + 'a|lib/c.txt': 'c', + 'a|$assetGraphPath': serializedGraph, + }, outputs: { + 'a|lib/a.txt.copy': 'c', + }, writer: writer); + + // Read cached graph and validate. + var graph = new AssetGraph.deserialize(JSON.decode( + UTF8.decode(writer.assets[makeAssetId('a|$assetGraphPath')])) as Map); + var outputNode = + graph.get(makeAssetId('a|lib/a.txt.copy')) as GeneratedAssetNode; + var aTxtNode = graph.get(makeAssetId('a|lib/a.txt')); + var bTxtNode = graph.get(makeAssetId('a|lib/b.txt')); + var cTxtNode = graph.get(makeAssetId('a|lib/c.txt')); + expect(outputNode.inputs, unorderedEquals([aTxtNode.id, cTxtNode.id])); + expect(aTxtNode.outputs, contains(outputNode.id)); + expect(bTxtNode.outputs, isEmpty); + expect(cTxtNode.outputs, unorderedEquals([outputNode.id])); }); }); } diff --git a/build_runner/test/generate/watch_test.dart b/build_runner/test/generate/watch_test.dart index ddbb3f9e8..4bdbc3eda 100644 --- a/build_runner/test/generate/watch_test.dart +++ b/build_runner/test/generate/watch_test.dart @@ -135,13 +135,15 @@ void main() { var expectedGraph = await AssetGraph.build([], new Set(), 'a', null); var bCopyNode = new GeneratedAssetNode(null, makeAssetId('a|web/b.txt'), false, true, makeAssetId('a|web/b.txt.copy'), - lastKnownDigest: computeDigest('b2')); + lastKnownDigest: computeDigest('b2'), + inputs: [makeAssetId('a|web/b.txt')]); expectedGraph.add(bCopyNode); expectedGraph.add( makeAssetNode('a|web/b.txt', [bCopyNode.id], computeDigest('b2'))); var cCopyNode = new GeneratedAssetNode(null, makeAssetId('a|web/c.txt'), false, true, makeAssetId('a|web/c.txt.copy'), - lastKnownDigest: computeDigest('c')); + lastKnownDigest: computeDigest('c'), + inputs: [makeAssetId('a|web/c.txt')]); expectedGraph.add(cCopyNode); expectedGraph.add( makeAssetNode('a|web/c.txt', [cCopyNode.id], computeDigest('c')));