From 197565a86ab436b6a8b5b9ec3bbb87f3b58cc1fc Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 5 Jul 2022 12:10:43 +0000 Subject: [PATCH 01/51] content hashing of arhcives --- lib/src/command/cache_add.dart | 2 +- lib/src/command/dependency_services.dart | 50 ++- lib/src/command/get.dart | 10 + lib/src/command/outdated.dart | 6 +- lib/src/entrypoint.dart | 75 ++-- lib/src/global_packages.dart | 11 +- lib/src/io.dart | 2 +- lib/src/lock_file.dart | 135 +++++- lib/src/null_safety_analysis.dart | 6 +- lib/src/solver/package_lister.dart | 6 +- lib/src/solver/report.dart | 12 +- lib/src/solver/result.dart | 44 +- lib/src/source.dart | 5 +- lib/src/source/cached.dart | 6 +- lib/src/source/git.dart | 17 +- lib/src/source/hosted.dart | 398 +++++++++++++----- lib/src/system_cache.dart | 38 +- lib/src/utils.dart | 47 +++ test/content_hash_test.dart | 193 +++++++++ .../dependency_services_test.dart | 16 +- test/get/enforce_lockfile_test.dart | 109 +++++ .../preserve_lock_file_line_endings_test.dart | 2 +- test/hosted/short_syntax_test.dart | 1 + test/lock_file_test.dart | 24 +- test/package_server.dart | 44 +- test/pubspec_test.dart | 30 +- test/reformat_ranges_test.dart | 1 + test/test_pub.dart | 50 +-- .../Adding transitive.txt | 27 +- .../dependency_services_test/Compatible.txt | 72 +++- .../No pubspec.lock.txt | 30 +- .../Relative paths are allowed.txt | 14 +- .../Removing transitive.txt | 33 +- .../multibreaking.txt | 72 +++- ...--verbose and on unexpected exceptions.txt | 9 + .../goldens/help_test/pub get --help.txt | 1 + test/utils_test.dart | 14 + 37 files changed, 1247 insertions(+), 365 deletions(-) create mode 100644 test/content_hash_test.dart create mode 100644 test/get/enforce_lockfile_test.dart diff --git a/lib/src/command/cache_add.dart b/lib/src/command/cache_add.dart index ea8a8c162..3c79d1d5c 100644 --- a/lib/src/command/cache_add.dart +++ b/lib/src/command/cache_add.dart @@ -77,7 +77,7 @@ class CacheAddCommand extends PubCommand { } // Download it. - await cache.downloadPackage(id); + await cache.downloadPackage(id, allowOutdatedHashChecks: true); } if (argResults['all']) { diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index 193f34513..527358425 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -24,6 +24,7 @@ import '../pubspec.dart'; import '../pubspec_utils.dart'; import '../solver.dart'; import '../source/git.dart'; +import '../source/hosted.dart'; import '../system_cache.dart'; import '../utils.dart'; @@ -394,6 +395,11 @@ class DependencyServicesApplyCommand extends PubCommand { lockFileYaml['packages'].containsKey(targetPackage)) { lockFileEditor.update( ['packages', targetPackage, 'version'], targetVersion.toString()); + // Remove the now outdated content-hash - it will be restored below + // after resolution. + lockFileEditor.remove( + ['packages', targetPackage, 'description', 'sha256'], + ); } else if (targetRevision != null && lockFileYaml['packages'].containsKey(targetPackage)) { final ref = entrypoint.lockFile.packages[targetPackage]!.toRef(); @@ -457,8 +463,48 @@ class DependencyServicesApplyCommand extends PubCommand { writeTextFile(entrypoint.pubspecPath, updatedPubspec); } // Only if we originally had a lock-file we write the resulting lockfile back. - if (lockFileEditor != null) { - entrypoint.saveLockFile(solveResult); + if (updatedLockfile != null) { + final updatedPackages = []; + for (final package in solveResult.packages) { + if (package.isRoot) continue; + final description = package.description; + if (description is ResolvedHostedDescription && + description.sha256 == null) { + // We removed the hash above before resolution - as we get the + // locked id back we need to find the content-hash from the + // version listing. + // + // `pub get` gets this version-listing from the downloaded archive + // but we don't want to download all archives - so we copy it from + // the version listing. + final listedId = (await cache.getVersions(package.toRef())) + .firstWhere((id) => id == package); + if ((listedId.description as ResolvedHostedDescription).sha256 == + null) { + // This happens when we resolved a package from a server not + // providing archive_sha256. As a side-effect of downloading + // the package we compute and store the sha256, and that will + // be picked up by entrypoint.saveLockFile. + await cache.downloadPackage( + package, + allowOutdatedHashChecks: false, + ); + } + updatedPackages.add(listedId); + } else { + updatedPackages.add(package); + } + } + + final newLockFile = LockFile( + updatedPackages, + sdkConstraints: updatedLockfile.sdkConstraints, + mainDependencies: pubspec.dependencies.keys.toSet(), + devDependencies: pubspec.devDependencies.keys.toSet(), + overriddenDependencies: pubspec.dependencyOverrides.keys.toSet(), + ); + + newLockFile.writeToFile(entrypoint.lockFilePath, cache); } }, ); diff --git a/lib/src/command/get.dart b/lib/src/command/get.dart index 7b8906cb7..f6a7b71f9 100644 --- a/lib/src/command/get.dart +++ b/lib/src/command/get.dart @@ -7,6 +7,7 @@ import 'dart:async'; import '../command.dart'; import '../log.dart' as log; import '../solver.dart'; +import '../utils.dart'; /// Handles the `get` pub command. class GetCommand extends PubCommand { @@ -28,6 +29,9 @@ class GetCommand extends PubCommand { negatable: false, help: "Report what dependencies would change but don't change any."); + argParser.addFlag('enforce-lockfile', + negatable: false, help: 'Only use resolution in existing lockfile.'); + argParser.addFlag('precompile', help: 'Build executables in immediate dependencies.'); @@ -52,12 +56,17 @@ class GetCommand extends PubCommand { log.warning(log.yellow( 'The --packages-dir flag is no longer used and does nothing.')); } + + if (argResults['dry-run'] && argResults['enforce-lockfile']) { + fail('Cannot do a dry-run with enforce-lockfile.'); + } await entrypoint.acquireDependencies( SolveType.get, dryRun: argResults['dry-run'], precompile: argResults['precompile'], generateDotPackages: argResults['legacy-packages-file'], analytics: analytics, + enforceLockfile: argResults['enforce-lockfile'], ); var example = entrypoint.example; @@ -69,6 +78,7 @@ class GetCommand extends PubCommand { generateDotPackages: argResults['legacy-packages-file'], analytics: analytics, onlyReportSuccessOrFailure: true, + enforceLockfile: argResults['enforce-lockfile'], ); } } diff --git a/lib/src/command/outdated.dart b/lib/src/command/outdated.dart index 601bdc8ff..3922737cc 100644 --- a/lib/src/command/outdated.dart +++ b/lib/src/command/outdated.dart @@ -211,7 +211,11 @@ class OutdatedCommand extends PubCommand { latestIsOverridden = true; } - final packageStatus = await current?.source.status(current, cache); + final packageStatus = await current?.source.status( + current.toRef(), + current.version, + cache, + ); final discontinued = packageStatus == null ? false : packageStatus.isDiscontinued; final discontinuedReplacedBy = packageStatus?.discontinuedReplacedBy; diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 3a51b2f32..fe82d1e6d 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -8,7 +8,6 @@ import 'dart:io'; import 'dart:math'; import 'package:collection/collection.dart'; -import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:pool/pool.dart'; import 'package:pub_semver/pub_semver.dart'; @@ -250,11 +249,11 @@ class Entrypoint { /// /// Performs version resolution according to [SolveType]. /// - /// [useLatest], if provided, defines a list of packages that will be - /// unlocked and forced to their latest versions. If [upgradeAll] is - /// true, the previous lockfile is ignored and all packages are re-resolved - /// from scratch. Otherwise, it will attempt to preserve the versions of all - /// previously locked packages. + /// [useLatest], if provided, defines a list of packages that will be unlocked + /// and forced to their latest versions. If [upgradeAll] is true, the previous + /// lockfile is ignored and all packages are re-resolved from scratch. + /// Otherwise, it will attempt to preserve the versions of all previously + /// locked packages. /// /// Shows a report of the changes made relative to the previous lockfile. If /// this is an upgrade or downgrade, all transitive dependencies are shown in @@ -264,10 +263,14 @@ class Entrypoint { /// If [precompile] is `true` (the default), this snapshots dependencies' /// executables. /// - /// if [onlyReportSuccessOrFailure] is `true` only success or failure will be shown --- - /// in case of failure, a reproduction command is shown. + /// if [onlyReportSuccessOrFailure] is `true` only success or failure will be + /// shown --- in case of failure, a reproduction command is shown. /// /// Updates [lockFile] and [packageRoot] accordingly. + /// + /// If [enforceLockfile] is true no changes to the current lockfile are + /// allowed. Instead the existing lockfile is loaded, verified against + /// pubspec.yaml and all dependencies downloaded. Future acquireDependencies( SolveType type, { Iterable? unlock, @@ -276,7 +279,13 @@ class Entrypoint { required bool generateDotPackages, required PubAnalytics? analytics, bool onlyReportSuccessOrFailure = false, + bool enforceLockfile = false, }) async { + if (enforceLockfile && !fileExists(lockFilePath)) { + throw ApplicationException( + 'Retrieving dependencies failed. Cannot do `--enforce-lockfile` without an existing `pubspec.lock`.'); + } + if (!onlyReportSuccessOrFailure && hasPubspecOverrides) { log.warning( 'Warning: pubspec.yaml has overrides from $pubspecOverridesPath'); @@ -306,6 +315,7 @@ class Entrypoint { rethrow; } } + _lockFile = result.lockFile; // Log once about all overridden packages. if (warnAboutPreReleaseSdkOverrides) { @@ -324,13 +334,24 @@ class Entrypoint { 'by setting it to `quiet`.')); } } + if (enforceLockfile) { + await result.enforceLockfile(); + } if (!onlyReportSuccessOrFailure) { await result.showReport(type, cache); } if (!dryRun) { - await result.downloadCachedPackages(cache); - saveLockFile(result); + await cache.downloadPackages( + root, + result.packages, + allowOutdatedHashChecks: !enforceLockfile, + ); + if (enforceLockfile) { + result.lockFile.enforceContentHashes(cache); + } else { + result.lockFile.writeToFile(lockFilePath, cache); + } } if (onlyReportSuccessOrFailure) { log.message('Got dependencies$suffix.'); @@ -874,21 +895,6 @@ class Entrypoint { } } - /// Saves a list of concrete package versions to the `pubspec.lock` file. - /// - /// Will use Windows line endings (`\r\n`) if a `pubspec.lock` exists, and - /// uses that. - void saveLockFile(SolveResult result) { - _lockFile = result.lockFile; - - final windowsLineEndings = fileExists(lockFilePath) && - detectWindowsLineEndings(readTextFile(lockFilePath)); - - final serialized = lockFile.serialize(root.dir); - writeTextFile(lockFilePath, - windowsLineEndings ? serialized.replaceAll('\n', '\r\n') : serialized); - } - /// If the entrypoint uses the old-style `.pub` cache directory, migrates it /// to the new-style `.dart_tool/pub` directory. void migrateCache() { @@ -962,22 +968,3 @@ See https://dart.dev/go/sdk-constraint } } } - -/// Returns `true` if the [text] looks like it uses windows line endings. -/// -/// The heuristic used is to count all `\n` in the text and if stricly more than -/// half of them are preceded by `\r` we report `true`. -@visibleForTesting -bool detectWindowsLineEndings(String text) { - var index = -1; - var unixNewlines = 0; - var windowsNewlines = 0; - while ((index = text.indexOf('\n', index + 1)) != -1) { - if (index != 0 && text[index - 1] == '\r') { - windowsNewlines++; - } else { - unixNewlines++; - } - } - return windowsNewlines > unixNewlines; -} diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index 8bfe7cf58..15db8262f 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -179,7 +179,7 @@ class GlobalPackages { final tempDir = cache.createTempDir(); // TODO(rnystrom): Look in "bin" and display list of binaries that // user can run. - _writeLockFile(tempDir, LockFile([id])); + _writeLockFile(tempDir, LockFile([id]), cache); tryDeleteEntry(_packageDir(name)); tryRenameDir(tempDir, _packageDir(name)); @@ -237,11 +237,12 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam // Only precompile binaries if we have a new resolution. if (!silent) await result.showReport(SolveType.get, cache); - await result.downloadCachedPackages(cache); + await cache.downloadPackages(root, result.packages, + allowOutdatedHashChecks: true); final lockFile = result.lockFile; final tempDir = cache.createTempDir(); - _writeLockFile(tempDir, lockFile); + _writeLockFile(tempDir, lockFile, cache); // Load the package graph from [result] so we don't need to re-parse all // the pubspecs. @@ -278,8 +279,8 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam } /// Finishes activating package [package] by saving [lockFile] in the cache. - void _writeLockFile(String dir, LockFile lockFile) { - writeTextFile(p.join(dir, 'pubspec.lock'), lockFile.serialize(null)); + void _writeLockFile(String dir, LockFile lockFile, SystemCache cache) { + writeTextFile(p.join(dir, 'pubspec.lock'), lockFile.serialize(null, cache)); } /// Shows the user the currently active package with [name], if any. diff --git a/lib/src/io.dart b/lib/src/io.dart index 50f7eb9d3..2b5d0cdc5 100644 --- a/lib/src/io.dart +++ b/lib/src/io.dart @@ -172,7 +172,7 @@ List readBinaryFile(String file) { } /// Reads the contents of the binary file [file] as a [Stream]. -Stream> readBinaryFileAsSream(String file) { +Stream> readBinaryFileAsStream(String file) { log.io('Reading binary file $file.'); var contents = File(file).openRead(); return contents; diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 215ab928e..3f0e71ea9 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -3,8 +3,10 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:convert'; +import 'dart:typed_data'; import 'package:collection/collection.dart' hide mapMap; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; import 'package:source_span/source_span.dart'; @@ -12,10 +14,12 @@ import 'package:yaml/yaml.dart'; import 'io.dart'; import 'language_version.dart'; +import 'log.dart' as log; import 'package_config.dart'; import 'package_name.dart'; import 'packages_file.dart' as packages_file; import 'sdk.dart' show sdk; +import 'source/hosted.dart'; import 'system_cache.dart'; import 'utils.dart'; @@ -30,15 +34,15 @@ class LockFile { /// Dependency names that appeared in the root package's `dependencies` /// section. - final Set _mainDependencies; + final Set mainDependencies; /// Dependency names that appeared in the root package's `dev_dependencies` /// section. - final Set _devDependencies; + final Set devDependencies; /// Dependency names that appeared in the root package's /// `dependency_overrides` section. - final Set _overriddenDependencies; + final Set overriddenDependencies; /// Creates a new lockfile containing [ids]. /// @@ -60,20 +64,16 @@ class LockFile { devDependencies ?? const UnmodifiableSetView.empty(), overriddenDependencies ?? const UnmodifiableSetView.empty()); - LockFile._( - Map packages, - this.sdkConstraints, - this._mainDependencies, - this._devDependencies, - this._overriddenDependencies) + LockFile._(Map packages, this.sdkConstraints, + this.mainDependencies, this.devDependencies, this.overriddenDependencies) : packages = UnmodifiableMapView(packages); LockFile.empty() : packages = const {}, sdkConstraints = {'dart': VersionConstraint.any}, - _mainDependencies = const UnmodifiableSetView.empty(), - _devDependencies = const UnmodifiableSetView.empty(), - _overriddenDependencies = const UnmodifiableSetView.empty(); + mainDependencies = const UnmodifiableSetView.empty(), + devDependencies = const UnmodifiableSetView.empty(), + overriddenDependencies = const UnmodifiableSetView.empty(); /// Loads a lockfile from [filePath]. factory LockFile.load(String filePath, SourceRegistry sources) { @@ -210,8 +210,13 @@ class LockFile { var packages = Map.from(this.packages); packages.remove(name); - return LockFile._(packages, sdkConstraints, _mainDependencies, - _devDependencies, _overriddenDependencies); + return LockFile._( + packages, + sdkConstraints, + mainDependencies, + devDependencies, + overriddenDependencies, + ); } /// Returns the contents of the `.packages` file generated from this lockfile. @@ -308,22 +313,67 @@ Generated by pub on ${DateTime.now()}.'''; return '${JsonEncoder.withIndent(' ').convert(packageConfig.toJson())}\n'; } + /// Throws a [DataException] if the content-hash of some hosted package locked in this lock file differs from the + /// one in the cache. + void enforceContentHashes(SystemCache cache) { + packages.forEach((name, package) { + var description = package.description; + if (description is ResolvedHostedDescription) { + if (description.sha256 != null) { + Uint8List? cachedHash = + description.description.source.sha256FromCache(package, cache); + if (cachedHash == null) { + // This can happen if we resolve from a server not providing hashes, + // but we are not downloading the archives. (eg. for a + // dependency_services run). + // TODO(sigurdm): What should we do here? + log.fine('No hash of downloaded archive found'); + } else if (!bytesEquals(cachedHash, description.sha256)) { + dataError( + 'Cache entry for $name-${package.version} does not have content-hash matching lockfile.'); + } + } + } + }); + } + /// Returns the serialized YAML text of the lock file. /// /// [packageDir] is the containing directory of the root package, used to /// serialize relative path package descriptions. If it is null, they will be /// serialized as absolute. - String serialize(String? packageDir) { + String serialize(String? packageDir, SystemCache cache) { // Convert the dependencies to a simple object. var packageMap = {}; packages.forEach((name, package) { - var description = - package.description.serializeForLockfile(containingDir: packageDir); + var description = package.description; + if (description is ResolvedHostedDescription) { + Uint8List? hash = + description.description.source.sha256FromCache(package, cache); + if (hash == null) { + // This can happen if we resolve from a server not providing hashes, + // but we are not downloading the archives. (eg. for a + // dependency_services run). + // TODO(sigurdm): What should we do here? + log.fine('No hash of downloaded archive found'); + } else if (description.sha256 == null) { + // We have resolved from a server without archive_sha256 in the + // version listing. Use the sha256 from the archive instead. + description = description.withSha256(hash); + } else { + // TODO(Should we really not fail here???). + if (!bytesEquals(hash, description.sha256)) { + dataError( + 'Cache entry for $name-${package.version} does not have content-hash matching lockfile.'); + } + } + } packageMap[name] = { 'version': package.version.toString(), 'source': package.source.name, - 'description': description, + 'description': + description.serializeForLockfile(containingDir: packageDir), 'dependency': _dependencyType(package.name) }; }); @@ -340,17 +390,37 @@ ${yamlToString(data)} '''; } + /// Saves the list of concrete package versions to [lockFilePath]. + /// + /// Will use Windows line endings (`\r\n`) if the file already exists, and + /// uses that. + /// + /// Relative paths will be resolved relative to [lockFilePath] + void writeToFile(String lockFilePath, SystemCache cache) { + final windowsLineEndings = fileExists(lockFilePath) && + detectWindowsLineEndings(readTextFile(lockFilePath)); + + final serialized = serialize(p.dirname(lockFilePath), cache); + writeTextFile(lockFilePath, + windowsLineEndings ? serialized.replaceAll('\n', '\r\n') : serialized); + } + + static const directMain = 'direct main'; + static const directDev = 'direct dev'; + static const directOverridden = 'direct overridden'; + static const transitive = 'transitive'; + /// Returns the dependency classification for [package]. String _dependencyType(String package) { - if (_mainDependencies.contains(package)) return 'direct main'; - if (_devDependencies.contains(package)) return 'direct dev'; + if (mainDependencies.contains(package)) return directMain; + if (devDependencies.contains(package)) return directDev; // If a package appears in `dependency_overrides` and another dependency // section, the main section it appears in takes precedence. - if (_overriddenDependencies.contains(package)) { - return 'direct overridden'; + if (overriddenDependencies.contains(package)) { + return directOverridden; } - return 'transitive'; + return transitive; } /// `true` if [other] has the same packages as `this` in the same versions @@ -366,3 +436,22 @@ ${yamlToString(data)} return true; } } + +/// Returns `true` if the [text] looks like it uses windows line endings. +/// +/// The heuristic used is to count all `\n` in the text and if stricly more than +/// half of them are preceded by `\r` we report `true`. +@visibleForTesting +bool detectWindowsLineEndings(String text) { + var index = -1; + var unixNewlines = 0; + var windowsNewlines = 0; + while ((index = text.indexOf('\n', index + 1)) != -1) { + if (index != 0 && text[index - 1] == '\r') { + windowsNewlines++; + } else { + unixNewlines++; + } + } + return windowsNewlines > unixNewlines; +} diff --git a/lib/src/null_safety_analysis.dart b/lib/src/null_safety_analysis.dart index fe89b559d..450098450 100644 --- a/lib/src/null_safety_analysis.dart +++ b/lib/src/null_safety_analysis.dart @@ -171,7 +171,11 @@ class NullSafetyAnalysis { if (source is CachedSource) { // TODO(sigurdm): Consider using withDependencyType here. - await source.downloadToSystemCache(dependencyId, _systemCache); + await source.downloadToSystemCache( + dependencyId, + _systemCache, + allowOutdatedHashChecks: true, + ); } final libDir = diff --git a/lib/src/solver/package_lister.dart b/lib/src/solver/package_lister.dart index 11988646e..e59793583 100644 --- a/lib/src/solver/package_lister.dart +++ b/lib/src/solver/package_lister.dart @@ -140,9 +140,9 @@ class PackageLister { /// /// Throws a [PackageNotFoundException] if this lister's package doesn't /// exist. - Future bestVersion(VersionConstraint? constraint) async { + Future bestVersion(VersionConstraint constraint) async { final locked = _locked; - if (locked != null && constraint!.allows(locked.version)) return locked; + if (locked != null && constraint.allows(locked.version)) return locked; var versions = await _versions; @@ -166,7 +166,7 @@ class PackageLister { for (var id in _isDowngrade ? versions : versions.reversed) { if (isPastLimit(id.version)) break; - if (!constraint!.allows(id.version)) continue; + if (!constraint.allows(id.version)) continue; if (!id.version.isPreRelease) { return id; } diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index dab29c63c..82555f454 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -148,8 +148,8 @@ class SolveReport { var numDiscontinued = 0; for (var id in _result.packages) { if (id.description is RootDescription) continue; - final status = - await id.source.status(id, _cache, maxAge: Duration(days: 3)); + final status = await id.source + .status(id.toRef(), id.version, _cache, maxAge: Duration(days: 3)); if (status.isDiscontinued && (_root.dependencyType(id.name) == DependencyType.direct || _root.dependencyType(id.name) == DependencyType.dev)) { @@ -259,8 +259,12 @@ class SolveReport { } } } - final status = - await id.source.status(id, _cache, maxAge: Duration(days: 3)); + final status = await id.source.status( + id.toRef(), + id.version, + _cache, + maxAge: Duration(days: 3), + ); if (status.isRetracted) { if (newerStable) { diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 7d57c36a7..abfb8ba45 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -5,7 +5,7 @@ import 'package:collection/collection.dart'; import 'package:pub_semver/pub_semver.dart'; -import '../http.dart'; +import '../exceptions.dart'; import '../io.dart'; import '../lock_file.dart'; import '../log.dart' as log; @@ -13,9 +13,9 @@ import '../package.dart'; import '../package_name.dart'; import '../pub_embeddable_command.dart'; import '../pubspec.dart'; -import '../source/cached.dart'; import '../source/hosted.dart'; import '../system_cache.dart'; +import '../utils.dart'; import 'report.dart'; import 'type.dart'; @@ -77,17 +77,6 @@ class SolveResult { final LockFile _previousLockFile; - /// Downloads all cached packages in [packages]. - Future downloadCachedPackages(SystemCache cache) async { - await Future.wait(packages.map((id) async { - final source = id.source; - if (source is! CachedSource) return; - return await withDependencyType(_root.dependencyType(id.name), () async { - await source.downloadToSystemCache(id, cache); - }); - })); - } - /// Returns the names of all packages that were changed. /// /// This includes packages that were added or removed. @@ -105,6 +94,35 @@ class SolveResult { SolveResult(this._root, this._previousLockFile, this.packages, this.pubspecs, this.availableVersions, this.attemptedSolutions, this.resolutionTime); + /// Checks that the SolveResult is compatible with [_previousLockfile] + /// + /// Throws if pubspec.yaml isn't satisfied. + Future enforceLockfile() async { + for (final package in packages) { + if (package.isRoot) continue; + final previousPackage = _previousLockFile.packages[package.name]; + if (previousPackage == null) { + throw ApplicationException( + 'Dependency ${package.name} is not already locked in `pubspec.lock`.'); + } + if (previousPackage != package) { + throw ApplicationException( + 'Dependency ${package.name} is locked to $previousPackage in `pubspec.lock` but resolves to $package.'); + } + final previousDescription = previousPackage.description; + final newDescription = package.description; + + if (previousDescription is ResolvedHostedDescription) { + final newHash = (newDescription as ResolvedHostedDescription).sha256; + + if (!bytesEquals(previousDescription.sha256, newHash)) { + throw ApplicationException( + 'Dependency ${package.name} is locked to a different content-hash than what was resolved.'); + } + } + } + } + /// Displays a report of what changes were made to the lockfile. /// /// [type] is the type of version resolution that was run. diff --git a/lib/src/source.dart b/lib/src/source.dart index e331da145..7f908e226 100644 --- a/lib/src/source.dart +++ b/lib/src/source.dart @@ -133,7 +133,7 @@ abstract class Source { String doGetDirectory(PackageId id, SystemCache cache, {String? relativeFrom}); - /// Returns metadata about a given package. + /// Returns metadata about a given package-version. /// /// For remotely hosted packages, the information can be cached for up to /// [maxAge]. If [maxAge] is not given, the information is not cached. @@ -141,7 +141,8 @@ abstract class Source { /// In the case of offline sources, [maxAge] is not used, since information is /// per definition cached. Future status( - PackageId id, + PackageRef ref, + Version version, SystemCache cache, { Duration? maxAge, }) async { diff --git a/lib/src/source/cached.dart b/lib/src/source/cached.dart index 16a7313dd..af85b829d 100644 --- a/lib/src/source/cached.dart +++ b/lib/src/source/cached.dart @@ -55,7 +55,11 @@ abstract class CachedSource extends Source { dirExists(getDirectoryInCache(id, cache)); /// Downloads the package identified by [id] to the system cache. - Future downloadToSystemCache(PackageId id, SystemCache cache); + Future downloadToSystemCache( + PackageId id, + SystemCache cache, { + required bool allowOutdatedHashChecks, + }); /// Returns the [Package]s that have been downloaded to the system cache. List getCachedPackages(SystemCache cache); diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index cb3d3f8c8..8659f8f74 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -188,7 +188,7 @@ class GitSource extends CachedSource { /// /// This lets us avoid race conditions when getting multiple different /// packages from the same repository. - final _revisionCacheClones = {}; + final _revisionCacheClones = >{}; /// The paths to the canonical clones of repositories for which "git fetch" /// has already been run during this run of pub. @@ -298,7 +298,11 @@ class GitSource extends CachedSource { /// itself; each of the commit-specific directories are clones of a directory /// in `cache/`. @override - Future downloadToSystemCache(PackageId id, SystemCache cache) async { + Future downloadToSystemCache( + PackageId id, + SystemCache cache, { + required bool allowOutdatedHashChecks, + }) async { return await _pool.withResource(() async { final ref = id.toRef(); final description = ref.description; @@ -318,7 +322,8 @@ class GitSource extends CachedSource { var revisionCachePath = _revisionCachePath(id, cache); final path = description.path; - await _revisionCacheClones.putIfAbsent(revisionCachePath, () async { + return await _revisionCacheClones.putIfAbsent(revisionCachePath, + () async { if (!entryExists(revisionCachePath)) { await _clone(_repoCachePath(ref, cache), revisionCachePath); await _checkOut(revisionCachePath, resolvedRef); @@ -327,12 +332,6 @@ class GitSource extends CachedSource { _updatePackageList(revisionCachePath, path); } }); - - return Package.load( - id.name, - p.join(revisionCachePath, p.fromUri(path)), - cache.sources, - ); }); } diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index e7dc87063..ba37e107c 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -5,9 +5,12 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io' as io; +import 'dart:io'; +import 'dart:typed_data'; import 'package:collection/collection.dart' - show maxBy, IterableNullableExtension; + show IterableExtension, IterableNullableExtension, ListEquality, maxBy; +import 'package:crypto/crypto.dart'; import 'package:http/http.dart' as http; import 'package:path/path.dart' as p; import 'package:pub_semver/pub_semver.dart'; @@ -149,24 +152,6 @@ class HostedSource extends CachedSource { return PackageRef(name, d); } - /// Returns an ID for a hosted package named [name] at [version]. - /// - /// If [url] is passed, it's the URL of the pub server from which the package - /// should be downloaded. [url] most be normalized and validated using - /// [validateAndNormalizeHostedUrl]. - PackageId idFor( - String name, - Version version, { - String? url, - }) => - PackageId( - name, - version, - ResolvedHostedDescription( - HostedDescription(name, url ?? defaultUrl.toString()), - ), - ); - /// Ensures that [description] is a valid hosted package description. /// /// Simple hosted dependencies only consist of a plain string, which is @@ -196,7 +181,10 @@ class HostedSource extends CachedSource { return PackageId( name, version, - ResolvedHostedDescription(HostedDescription(name, defaultUrl)), + ResolvedHostedDescription( + HostedDescription(name, defaultUrl), + sha256: null, + ), ); } if (description is! Map) { @@ -206,6 +194,10 @@ class HostedSource extends CachedSource { if (url is! String) { throw FormatException('The url should be a string.'); } + final sha256 = description['sha256']; + if (sha256 != null && sha256 is! String) { + throw FormatException('The sha256 should be a string.'); + } final foundName = description['name']; if (foundName is! String) { throw FormatException('The name should be a string.'); @@ -218,6 +210,7 @@ class HostedSource extends CachedSource { version, ResolvedHostedDescription( HostedDescription(name, Uri.parse(url).toString()), + sha256: sha256 == null ? null : hexDecode(sha256), ), ); } @@ -294,48 +287,53 @@ class HostedSource extends CachedSource { static final RegExp _looksLikePackageName = RegExp(r'^[a-zA-Z_]+[a-zA-Z0-9_]*$'); - late final RateLimitedScheduler<_RefAndCache, Map?> - _scheduler = RateLimitedScheduler( + late final RateLimitedScheduler<_RefAndCache, List<_VersionInfo>> _scheduler = + RateLimitedScheduler( _fetchVersions, maxConcurrentOperations: 10, ); - Map _versionInfoFromPackageListing( + List<_VersionInfo> _versionInfoFromPackageListing( Map body, PackageRef ref, Uri location, SystemCache cache) { final description = ref.description; if (description is! HostedDescription) { throw ArgumentError('Wrong source'); } final versions = body['versions']; - if (versions is List) { - return Map.fromEntries(versions.map((map) { - final pubspecData = map['pubspec']; - if (pubspecData is Map) { - var pubspec = Pubspec.fromMap(pubspecData, cache.sources, - expectedName: ref.name, location: location); - var id = idFor( - ref.name, - pubspec.version, - url: description.url, - ); - var archiveUrl = map['archive_url']; - if (archiveUrl is String) { - final status = PackageStatus( - isDiscontinued: body['isDiscontinued'] ?? false, - discontinuedReplacedBy: body['replacedBy'], - isRetracted: map['retracted'] ?? false); - return MapEntry( - id, _VersionInfo(pubspec, Uri.parse(archiveUrl), status)); - } - throw FormatException('archive_url must be a String'); - } - throw FormatException('pubspec must be a map'); - })); + if (versions is! List) { + throw FormatException('versions must be a list'); } - throw FormatException('versions must be a list'); + return versions.map((map) { + final pubspecData = map['pubspec']; + if (pubspecData is! Map) { + throw FormatException('pubspec must be a map'); + } + var pubspec = Pubspec.fromMap(pubspecData, cache.sources, + expectedName: ref.name, location: location); + final archiveSha256 = map['archive_sha256']; + if (archiveSha256 != null && archiveSha256 is! String) { + throw FormatException('archive_sha256 must be a String'); + } + final archiveUrl = map['archive_url']; + if (archiveUrl is! String) { + throw FormatException('archive_url must be a String'); + } + final status = PackageStatus( + isDiscontinued: body['isDiscontinued'] ?? false, + discontinuedReplacedBy: body['replacedBy'], + isRetracted: map['retracted'] ?? false, + ); + return _VersionInfo( + pubspec.version, + pubspec, + Uri.parse(archiveUrl), + status, + archiveSha256 == null ? null : hexDecode(archiveSha256), + ); + }).toList(); } - Future?> _fetchVersionsNoPrefetching( + Future> _fetchVersionsNoPrefetching( PackageRef ref, SystemCache cache) async { final description = ref.description; @@ -346,9 +344,9 @@ class HostedSource extends CachedSource { final url = _listVersionsUrl(ref); log.io('Get versions from $url.'); - late final String bodyText; - late final dynamic body; - late final Map result; + final String bodyText; + final dynamic body; + final List<_VersionInfo> result; try { // TODO(sigurdm): Implement cancellation of requests. This probably // requires resolution of: https://github.com/dart-lang/sdk/issues/22265. @@ -376,8 +374,7 @@ class HostedSource extends CachedSource { return result; } - Future?> _fetchVersions( - _RefAndCache refAndCache) async { + Future> _fetchVersions(_RefAndCache refAndCache) async { final ref = refAndCache.ref; final description = ref.description; if (description is! HostedDescription) { @@ -389,16 +386,13 @@ class HostedSource extends CachedSource { /// Prefetch the dependencies of the latest version, we are likely to need /// them later. void prescheduleDependenciesOfLatest( - Map? listing, + List<_VersionInfo>? listing, SystemCache cache, ) { - if (listing == null) return; + if (listing == null || listing.isEmpty) return; final latestVersion = - maxBy(listing.keys.map((id) => id.version), (e) => e)!; - final latestVersionId = PackageId( - ref.name, latestVersion, ResolvedHostedDescription(description)); - final dependencies = - listing[latestVersionId]?.pubspec.dependencies.values ?? []; + maxBy<_VersionInfo, Version>(listing, (e) => e.version)!; + final dependencies = latestVersion.pubspec.dependencies.values; unawaited(withDependencyType(DependencyType.none, () async { for (final packageRange in dependencies) { if (packageRange.source is HostedSource) { @@ -431,8 +425,7 @@ class HostedSource extends CachedSource { /// Invariant: Entries in this cache are the parsed version of the exact same /// information cached on disk. I.e. if the entry is present in this cache, /// there will not be a newer version on disk. - final Map>> - _responseCache = {}; + final Map>> _responseCache = {}; /// If a cached version listing response for [ref] exists on disk and is less /// than [maxAge] old it is parsed and returned. @@ -441,7 +434,7 @@ class HostedSource extends CachedSource { /// /// If [maxAge] is not given, we will try to get the cached version no matter /// how old it is. - Future?> _cachedVersionListingResponse( + Future?> _cachedVersionListingResponse( PackageRef ref, SystemCache cache, {Duration? maxAge}) async { if (_responseCache.containsKey(ref)) { @@ -519,26 +512,39 @@ class HostedSource extends CachedSource { } @override - Future status(PackageId id, SystemCache cache, - {Duration? maxAge}) async { + Future status( + PackageRef ref, + Version version, + SystemCache cache, { + Duration? maxAge, + }) async { + // If we don't have the specific version we return the empty response, since + // it is more or less harmless.. + // + // This can happen if the connection is broken, or the server is faulty. + // We want to avoid a crash + // + // TODO(sigurdm): Consider representing the non-existence of the + // package-version in the return value. + return (await _versionInfo(ref, version, cache, maxAge: maxAge))?.status ?? + PackageStatus(); + } + + Future<_VersionInfo?> _versionInfo( + PackageRef ref, + Version version, + SystemCache cache, { + Duration? maxAge, + }) async { if (cache.isOffline) { // Do we have a cached version response on disk? - final versionListing = - await _cachedVersionListingResponse(id.toRef(), cache); + final versionListing = await _cachedVersionListingResponse(ref, cache); if (versionListing == null) { - return PackageStatus(); + return null; } - // If we don't have the specific version we return the empty response. - // - // This should not happen. But in production we want to avoid a crash, since - // it is more or less harmless. - // - // TODO(sigurdm): Consider representing the non-existence of the - // package-version in the return value. - return versionListing[id]?.status ?? PackageStatus(); + return versionListing.firstWhereOrNull((l) => l.version == version); } - final ref = id.toRef(); // Did we already get info for this package? var versionListing = _scheduler.peek(_RefAndCache(ref, cache)); if (maxAge != null) { @@ -551,20 +557,11 @@ class HostedSource extends CachedSource { .schedule(_RefAndCache(ref, cache)) // Failures retrieving the listing here should just be ignored. .catchError( - (_) => {}, + (_) async => <_VersionInfo>[], test: (error) => error is Exception, ); - final listing = versionListing![id]; - // If we don't have the specific version we return the empty response, since - // it is more or less harmless.. - // - // This can happen if the connection is broken, or the server is faulty. - // We want to avoid a crash - // - // TODO(sigurdm): Consider representing the non-existence of the - // package-version in the return value. - return listing?.status ?? PackageStatus(); + return versionListing.firstWhereOrNull((l) => l.version == version); } // The path where the response from the package-listing api is cached. @@ -627,7 +624,18 @@ class HostedSource extends CachedSource { await _cachedVersionListingResponse(ref, cache, maxAge: maxAge); } versionListing ??= await _scheduler.schedule(_RefAndCache(ref, cache)); - return versionListing!.keys.toList(); + return versionListing + .map( + (i) => PackageId( + ref.name, + i.version, + ResolvedHostedDescription( + ref.description as HostedDescription, + sha256: i.archiveSha256, + ), + ), + ) + .toList(); } /// Parses [description] into its server and package name components, then @@ -653,23 +661,83 @@ class HostedSource extends CachedSource { } final versions = await _scheduler.schedule(_RefAndCache(id.toRef(), cache)); final url = _listVersionsUrl(id.toRef()); - return versions![id]?.pubspec ?? + return versions.firstWhereOrNull((i) => i.version == id.version)?.pubspec ?? (throw PackageNotFoundException('Could not find package $id at $url')); } - /// Downloads the package identified by [id] to the system cache. + /// Downloads the package identified by [id] to the system cache if needed. + /// + /// Validates that the content hash of [id] corresponds to what is already in + /// cache, if not the file is redownloaded. @override - Future downloadToSystemCache(PackageId id, SystemCache cache) async { - if (!isInSystemCache(id, cache)) { + Future downloadToSystemCache( + PackageId id, + SystemCache cache, { + required bool allowOutdatedHashChecks, + }) async { + final packageDir = getDirectoryInCache(id, cache); + + // Use the content-hash from the version-info to compare with what we + // already downloaded. + // + // The content-hash from [id] will be compared with that when the lockfile + // is written. + // + // We allow the version-listing to be a few days outdated in order for `pub + // get` with an existing working resolution and everything in cache to be + // fast. + final versionInfo = await _versionInfo(id.toRef(), id.version, cache, + maxAge: allowOutdatedHashChecks ? Duration(days: 3) : null); + + final expectedContentHash = versionInfo?.archiveSha256 ?? + // Handling of legacy server - we use the hash from the id (typically + // from the lockfile) to compare to the existing download. + (id.description as ResolvedHostedDescription).sha256; + if (!fileExists(hashPath(id, cache))) { + if (dirExists(packageDir) && !cache.isOffline) { + log.fine( + 'Cache entry for ${id.name}-${id.version} has no content-hash - redownloading.'); + deleteEntry(packageDir); + } + } else if (expectedContentHash == null) { + log.fine( + 'Content-hash of ${id.name}-${id.version} not known from resolution.'); + } else { + if (!bytesEquals(sha256FromCache(id, cache), expectedContentHash)) { + log.warning( + 'Cached version of ${id.name}-${id.version} has wrong hash - redownloading.'); + if (cache.isOffline) { + fail('Cannot redownload while offline - try again online.'); + } + deleteEntry(packageDir); + } + } + if (!dirExists(packageDir)) { if (cache.isOffline) { throw StateError('Cannot download packages when offline.'); } - var packageDir = getDirectoryInCache(id, cache); - ensureDir(p.dirname(packageDir)); await _download(id, packageDir, cache); } + } - return Package.load(id.name, getDirectoryInCache(id, cache), cache.sources); + /// Determines if the package identified by [id] is already downloaded to the + /// system cache and has the expected content-hash. + @override + bool isInSystemCache(PackageId id, SystemCache cache) { + if ((id.description as ResolvedHostedDescription).sha256 != null) { + try { + final cachedSha256 = readTextFile(hashPath(id, cache)); + if (!const ListEquality().equals(hexDecode(cachedSha256), + (id.description as ResolvedHostedDescription).sha256)) { + return false; + } + } on io.IOException { + // Most likely the hash file was not written, because we had a legacy + // entry. + return false; + } + } + return dirExists(getDirectoryInCache(id, cache)); } /// The system cache directory for the hosted source contains subdirectories @@ -689,6 +757,31 @@ class HostedSource extends CachedSource { return p.join(rootDir, dir, '${id.name}-${id.version}'); } + /// The system cache directory for the hosted source contains subdirectories + /// for each separate repository URL that's used on the system. + /// + /// Each of these subdirectories then contains a `.hashes` directory with a + /// stored hash of all downloaded packages. + String hashPath(PackageId id, SystemCache cache) { + final description = id.description.description; + if (description is! HostedDescription) { + throw ArgumentError('Wrong source'); + } + final rootDir = cache.rootDirForSource(this); + + var dir = _urlToDirectory(description.url); + return p.join(rootDir, dir, '.hashes', '${id.name}-${id.version}.sha256'); + } + + /// Loads the hash at `hashPath(id)`. + Uint8List? sha256FromCache(PackageId id, SystemCache cache) { + try { + return hexDecode(readTextFile(hashPath(id, cache))); + } on IOException { + return null; + } + } + /// Re-downloads all packages that have been previously downloaded into the /// system cache from any server. @override @@ -741,7 +834,14 @@ class HostedSource extends CachedSource { return results ..addAll(await Future.wait( packages.map((package) async { - var id = idFor(package.name, package.version, url: url); + var id = PackageId( + package.name, + package.version, + ResolvedHostedDescription( + HostedDescription(package.name, url), + sha256: null, + ), + ); try { deleteEntry(package.dir); await _download(id, package.dir, cache); @@ -778,7 +878,7 @@ class HostedSource extends CachedSource { return PackageId( name, version, - ResolvedHostedDescription(HostedDescription(name, url)), + ResolvedHostedDescription(HostedDescription(name, url), sha256: null), ); } @@ -843,7 +943,8 @@ class HostedSource extends CachedSource { // query-string as is the case with signed S3 URLs. And we wish to allow for // such URLs to be used. final versions = await _scheduler.schedule(_RefAndCache(id.toRef(), cache)); - final versionInfo = versions![id]; + final versionInfo = + versions.firstWhereOrNull((i) => i.version == id.version); final packageName = id.name; final version = id.version; if (versionInfo == null) { @@ -864,15 +965,49 @@ class HostedSource extends CachedSource { Uri.parse(description.url), (client) => client.send(http.Request('GET', url))); + Stream> checkSha256( + Stream> stream, + Digest? expectedHash, + ) async* { + final output = _SingleValueSink(); + final input = sha256.startChunkedConversion(output); + await for (final v in stream) { + input.add(v); + yield v; + } + input.close(); + final actualHash = output.value; + if (expectedHash != null && output.value != expectedHash) { + log.fine( + 'Expected content-hash $expectedHash actual: ${output.value}.'); + throw FormatException( + 'Downloaded archive for ${id.name}-${id.version} had wrong content-hash.'); + } + final path = hashPath(id, cache); + ensureDir(p.dirname(path)); + writeTextFile( + path, + hexEncode(actualHash.bytes), + ); + } + // We download the archive to disk instead of streaming it directly into // the tar unpacking. This simplifies stream handling. // Package:tar cancels the stream when it reaches end-of-archive, and // cancelling a http stream makes it not reusable. // There are ways around this, and we might revisit this later. - await createFileFromStream(response.stream, archivePath); - var tempDir = cache.createTempDir(); - await extractTarGz(readBinaryFileAsSream(archivePath), tempDir); + Stream> stream = response.stream; + final expectedSha256 = versionInfo.archiveSha256; + stream = checkSha256( + stream, (expectedSha256 == null) ? null : Digest(expectedSha256)); + + await createFileFromStream(stream, archivePath); + final tempDir = cache.createTempDir(); + + await extractTarGz(readBinaryFileAsStream(archivePath), tempDir); + + ensureDir(p.dirname(destPath)); // Now that the get has succeeded, move it to the real location in the // cache. // @@ -1000,7 +1135,22 @@ class ResolvedHostedDescription extends ResolvedDescription { @override HostedDescription get description => super.description as HostedDescription; - ResolvedHostedDescription(HostedDescription description) : super(description); + /// The content hash of the package archive (the `tar.gz` file) of the + /// PackageId described by this. + /// + /// This can be obtained in several ways: + /// * Reported from a server in the archive_sha256 field. + /// (will be null if the server does not report this.) + /// * Obtained from a pubspec.lock + /// (will be null for legacy lock-files). + /// * Read from the /hosted/.hashes/-.sha256 file. + /// (will be null if the file doesn't exist for corrupt or legacy caches). + final Uint8List? sha256; + + ResolvedHostedDescription( + HostedDescription description, { + required this.sha256, + }) : super(description); @override Object? serializeForLockfile({required String? containingDir}) { @@ -1010,26 +1160,46 @@ class ResolvedHostedDescription extends ResolvedDescription { } on FormatException catch (e) { throw ArgumentError.value(url, 'url', 'url must be normalized: $e'); } - return {'name': description.packageName, 'url': url.toString()}; + final hash = sha256; + return { + 'name': description.packageName, + 'url': url.toString(), + if (hash != null) 'sha256': hexEncode(hash), + }; } @override + // We do not include the sha256 in the hashCode because of the equality + // semantics. int get hashCode => description.hashCode; @override bool operator ==(Object other) { return other is ResolvedHostedDescription && - other.description == description; + other.description == description && + // A [sha256] of `null` means that we don't know the hash yet. + // Therefore we have to assume it is equal to any known value. + (sha256 == null || + other.sha256 == null || + const ListEquality().equals(sha256, other.sha256)); } + + ResolvedHostedDescription withSha256(Uint8List? newSha256) => + ResolvedHostedDescription(description, sha256: newSha256); } /// Information about a package version retrieved from /api/packages/$package< class _VersionInfo { final Pubspec pubspec; final Uri archiveUrl; + final Version version; + + /// The sha256 digest of the archive according to the package-repository. + final Uint8List? archiveSha256; final PackageStatus status; - _VersionInfo(this.pubspec, this.archiveUrl, this.status); + _VersionInfo(this.version, this.pubspec, this.archiveUrl, this.status, + this.archiveSha256); } /// Given a URL, returns a "normalized" string to be used as a directory name @@ -1100,3 +1270,17 @@ class _RefAndCache { @override bool operator ==(Object other) => other is _RefAndCache && other.ref == ref; } + +/// A sink that can only have `add` called once, and that can retrieve the +/// value. +class _SingleValueSink implements Sink { + late final T value; + + @override + void add(T data) { + value = data; + } + + @override + void close() {} +} diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart index ebbaa9792..19bc8b250 100644 --- a/lib/src/system_cache.dart +++ b/lib/src/system_cache.dart @@ -10,6 +10,7 @@ import 'package:pub_semver/pub_semver.dart'; import 'authentication/token_store.dart'; import 'exceptions.dart'; +import 'http.dart'; import 'io.dart'; import 'io.dart' as io show createTempDir; import 'log.dart' as log; @@ -186,7 +187,12 @@ class SystemCache { var versions = await ref.source.doGetVersions(ref, maxAge, this); versions = (await Future.wait(versions.map((id) async { - final packageStatus = await ref.source.status(id, this, maxAge: maxAge); + final packageStatus = await ref.source.status( + id.toRef(), + id.version, + this, + maxAge: maxAge, + ); if (!packageStatus.isRetracted || id.version == allowedRetractedVersion) { return id; } @@ -208,10 +214,17 @@ class SystemCache { return id.source.doGetDirectory(id, this, relativeFrom: relativeFrom); } - Future downloadPackage(PackageId id) async { + Future downloadPackage( + PackageId id, { + required bool allowOutdatedHashChecks, + }) async { final source = id.source; assert(source is CachedSource); - await (source as CachedSource).downloadToSystemCache(id, this); + await (source as CachedSource).downloadToSystemCache( + id, + this, + allowOutdatedHashChecks: allowOutdatedHashChecks, + ); } /// Get the latest version of [package]. @@ -251,6 +264,25 @@ class SystemCache { return latest; } + + /// Downloads all cached packages in [packages]. + Future downloadPackages( + Package root, + List packages, { + required bool allowOutdatedHashChecks, + }) async { + await Future.wait(packages.map((id) async { + if (id.source is! CachedSource) { + return; + } + return await withDependencyType(root.dependencyType(id.name), () async { + await downloadPackage( + id, + allowOutdatedHashChecks: allowOutdatedHashChecks, + ); + }); + })); + } } typedef SourceRegistry = Source Function(String? name); diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 3e1cf4863..b2a08dd51 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -7,7 +7,9 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; +import 'dart:typed_data'; +import 'package:collection/collection.dart'; import 'package:crypto/crypto.dart' as crypto; import 'package:pub_semver/pub_semver.dart'; import 'package:stack_trace/stack_trace.dart'; @@ -328,6 +330,47 @@ String replace(String source, Pattern matcher, String Function(Match) fn) { String sha1(String source) => crypto.sha1.convert(utf8.encode(source)).toString(); +final _hexTable = [ + '0', '1', '2', '3', '4', '5', '6', '7', // + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', +]; + +final _hexTable2 = Map.fromIterables(_hexTable, List.generate(16, (i) => i)); + +String hexEncode(List bytes) { + const mask = (1 << 4) - 1; + final buffer = StringBuffer(); + for (final byte in bytes) { + if (byte > 255 || byte < 0) { + throw FormatException('Bad value in byte list $byte.'); + } + buffer.write(_hexTable[byte >> 4 & mask]); + buffer.write(_hexTable[byte & mask]); + } + return buffer.toString(); +} + +Uint8List hexDecode(String string) { + string = string.toLowerCase(); + if (string.length % 2 != 0) { + throw FormatException( + 'Bad hex encoding, must have an even number of characters'); + } + final result = Uint8List(string.length ~/ 2); + for (var i = 0; i < result.length; i++) { + final v = _hexTable2[string[i * 2]]; + if (v == null) { + throw FormatException('Bad char `${string[i * 2]}` in hex encoding'); + } + final v2 = _hexTable2[string[i * 2 + 1]]; + if (v2 == null) { + throw FormatException('Bad char `${string[i * 2 + 1]}` in hex encoding'); + } + result[i] = (v << 4) | v2; + } + return result; +} + /// A regular expression matching a trailing CR character. final _trailingCR = RegExp(r'\r$'); @@ -622,3 +665,7 @@ Map mapMap( key(entry.key, entry.value): value(entry.key, entry.value), }; } + +bool bytesEquals(List? a, List? b) { + return const ListEquality().equals(a, b); +} diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart new file mode 100644 index 000000000..56c100191 --- /dev/null +++ b/test/content_hash_test.dart @@ -0,0 +1,193 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:pub/src/exit_codes.dart' as exit_codes; +import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; + +import 'descriptor.dart'; +import 'test_pub.dart'; + +Future main() async { + test('archive_sha256 is stored in lockfile and cache upon download', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.serveContentHashes = true; + await appDir({'foo': 'any'}).create(); + await pubGet(); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final sha256 = lockfile['packages']['foo']['description']['sha256']; + expect(sha256, hasLength(64)); + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', sha256), + ]) + ]).validate(); + }); + + test( + 'archive_sha256 is stored in lockfile upon download on legacy server without content hashes', + () async { + final server = await servePackages(); + server.serveContentHashes = false; + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final sha256 = lockfile['packages']['foo']['description']['sha256']; + expect(sha256, hasLength(64)); + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', sha256), + ]) + ]).validate(); + }); + + test('archive_sha256 is checked on download', () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.setSha256('foo', '1.0.0', + 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'); + await appDir({'foo': 'any'}).create(); + await pubGet( + error: + contains('Downloaded archive for foo-1.0.0 had wrong content-hash.'), + exitCode: exit_codes.DATA, + ); + }); + + test('If content is updated on server we refuse to continue', () async { + final server = await servePackages(); + server.serveContentHashes = true; + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + server.serve('foo', '1.0.0', + contents: [file('new_file.txt', 'This file could be malicious.')]); + // Pub get will not revisit the file-listing if everything resolves, and only compare with a cached value. + await pubGet(); + // Deleting the version-listing cache will cause it to be refetched, and the error will happen. + File(p.join(globalServer.cachingPath, '.cache', 'foo-versions.json')) + .deleteSync(); + + await pubGet( + error: allOf( + contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), + contains( + 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.'), + ), + exitCode: exit_codes.DATA, + ); + }); + + test( + 'sha256 in cache is checked on pub get - warning and redownload on legacy server without content-hashes', + () async { + final server = await servePackages(); + server.serveContentHashes = false; + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final originalHash = lockfile['packages']['foo']['description']['sha256']; + // Create wrong hash on disk. + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', + 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), + ]) + ]).create(); + + await pubGet( + warning: 'Cached version of foo-1.0.0 has wrong hash - redownloading.'); + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', originalHash), + ]) + ]).validate(); + }); + + test('sha256 in cache is checked on pub get - warning and redownload', + () async { + final server = await servePackages(); + server.serveContentHashes = true; + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final originalHash = lockfile['packages']['foo']['description']['sha256']; + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', + 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), + ]) + ]).create(); + + await pubGet( + warning: 'Cached version of foo-1.0.0 has wrong hash - redownloading.'); + await hostedCache([ + dir('.hashes', [ + file('foo-1.0.0.sha256', originalHash), + ]) + ]).validate(); + }); + test( + 'Legacy lockfile without content-hashes is updated with the hash on pub get on legacy server without content-hashes', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.serveContentHashes = false; + await appDir({'foo': 'any'}).create(); + await pubGet(); + // Pretend we had no hash in the lockfile. + final lockfile = YamlEditor( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final originalContentHash = lockfile + .remove(['packages', 'foo', 'description', 'sha256']).value as String; + File(p.join(sandbox, appPath, 'pubspec.lock')).writeAsStringSync( + lockfile.toString(), + ); + await pubGet(); + final lockfile2 = YamlEditor( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + expect( + lockfile2.parseAt(['packages', 'foo', 'description', 'sha256']).value, + originalContentHash, + ); + }); + + test( + 'Legacy lockfile without content-hashes is updated with the hash on pub get', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.serveContentHashes = true; + await appDir({'foo': 'any'}).create(); + await pubGet(); + // Pretend we had no hash in the lockfile. + final lockfile = YamlEditor( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final originalContentHash = lockfile + .remove(['packages', 'foo', 'description', 'sha256']).value as String; + File(p.join(sandbox, appPath, 'pubspec.lock')).writeAsStringSync( + lockfile.toString(), + ); + await pubGet(); + final lockfile2 = YamlEditor( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + expect( + lockfile2.parseAt(['packages', 'foo', 'description', 'sha256']).value, + originalContentHash, + ); + }); +} diff --git a/test/dependency_services/dependency_services_test.dart b/test/dependency_services/dependency_services_test.dart index 7228b36a8..5832a6644 100644 --- a/test/dependency_services/dependency_services_test.dart +++ b/test/dependency_services/dependency_services_test.dart @@ -49,6 +49,7 @@ extension on GoldenTestContext { Platform.resolvedExecutable, [ snapshot, + '--verbose', ...args, ], environment: getPubTestEnvironment(), @@ -120,7 +121,8 @@ Future main() async { final server = (await servePackages()) ..serve('foo', '1.2.3', deps: {'transitive': '^1.0.0'}) ..serve('foo', '2.2.3') - ..serve('transitive', '1.0.0'); + ..serve('transitive', '1.0.0') + ..serveContentHashes = true; await d.dir(appPath, [ d.pubspec({ @@ -151,7 +153,8 @@ Future main() async { final server = (await servePackages()) ..serve('foo', '1.2.3', deps: {'transitive': '^1.0.0'}) ..serve('foo', '2.2.3') - ..serve('transitive', '1.0.0'); + ..serve('transitive', '1.0.0') + ..serveContentHashes = true; await d.git('bar.git', [d.libPubspec('bar', '1.0.0')]).create(); @@ -183,7 +186,8 @@ Future main() async { ..serve('foo', '2.2.3') ..serve('bar', '1.2.3') ..serve('bar', '2.2.3') - ..serve('boo', '1.2.3'); + ..serve('boo', '1.2.3') + ..serveContentHashes = true; await d.dir(appPath, [ d.pubspec({ @@ -215,7 +219,8 @@ Future main() async { final server = (await servePackages()) ..serve('foo', '1.2.3') ..serve('foo', '2.2.3', deps: {'transitive': '^1.0.0'}) - ..serve('transitive', '1.0.0'); + ..serve('transitive', '1.0.0') + ..serveContentHashes = true; await d.dir(appPath, [ d.pubspec({ @@ -247,7 +252,8 @@ Future main() async { final server = (await servePackages()) ..serve('foo', '1.0.0') ..serve('bar', '1.0.0') - ..serve('baz', '1.0.0'); + ..serve('baz', '1.0.0') + ..serveContentHashes = true; await d.dir(appPath, [ d.pubspec({ diff --git a/test/get/enforce_lockfile_test.dart b/test/get/enforce_lockfile_test.dart new file mode 100644 index 000000000..3f9f41fb1 --- /dev/null +++ b/test/get/enforce_lockfile_test.dart @@ -0,0 +1,109 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:pub/src/exit_codes.dart'; +import 'package:test/test.dart'; + +import '../descriptor.dart'; +import '../test_pub.dart'; + +Future main() async { + test('Recreates .dart_tool/package_config.json, redownloads archives', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + final packageConfig = + File(path(p.join(appPath, '.dart_tool', 'package_config.json'))); + packageConfig.deleteSync(); + await runPub(args: ['cache', 'clean', '-f']); + await pubGet(args: ['--enforce-lockfile']); + expect(packageConfig.existsSync(), isTrue); + await cacheDir({'foo': '1.0.0'}).validate(); + await appPackageConfigFile([ + packageConfigEntry(name: 'foo', version: '1.0.0'), + ]).validate(); + }); + + test('Refuses to get if no lockfile exists', () async { + await appDir({}).create(); + await pubGet( + args: ['--enforce-lockfile'], + error: + 'Retrieving dependencies failed. Cannot do `--enforce-lockfile` without an existing `pubspec.lock`.'); + }); + + test('Refuses to get if lockfile is missing package', () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + await appDir({}).create(); + await pubGet(); + await appDir({'foo': 'any'}).create(); + + await pubGet( + args: ['--enforce-lockfile'], + error: 'Dependency foo is not already locked in `pubspec.lock`.'); + }); + + test('Refuses to get if package is locked to version not matching constraint', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.serve('foo', '2.0.0'); + await appDir({'foo': '^1.0.0'}).create(); + await pubGet(); + await appDir({'foo': '^2.0.0'}).create(); + await pubGet( + args: ['--enforce-lockfile'], + error: + 'Dependency foo is locked to foo 1.0.0 in `pubspec.lock` but resolves to foo 2.0.0.'); + }); + + test("Refuses to get if hash on server doesn't correspond to lockfile", + () async { + final server = await servePackages(); + server.serveContentHashes = true; + server.serve('foo', '1.0.0'); + await appDir({'foo': '^1.0.0'}).create(); + await pubGet(); + server.serve('foo', '1.0.0', contents: [ + file('README.md', 'Including this will change the content-hash.'), + ]); + + await pubGet( + args: ['--enforce-lockfile'], + error: allOf( + contains( + 'Cached version of foo-1.0.0 has wrong hash - redownloading.'), + contains( + 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.')), + exitCode: DATA, + ); + }); + + test( + 'Refuses to get if archive on legacy server doesn\'t have hash corresponding to lockfile', + () async { + final server = await servePackages(); + server.serveContentHashes = false; + server.serve('foo', '1.0.0'); + await appDir({'foo': '^1.0.0'}).create(); + await pubGet(); + await runPub(args: ['cache', 'clean', '-f']); + server.serve('foo', '1.0.0', contents: [ + file('README.md', 'Including this will change the content-hash.'), + ]); + + await pubGet( + args: ['--enforce-lockfile'], + error: + 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.', + exitCode: DATA, + ); + }); +} diff --git a/test/get/preserve_lock_file_line_endings_test.dart b/test/get/preserve_lock_file_line_endings_test.dart index bfd015a83..4484fc55e 100644 --- a/test/get/preserve_lock_file_line_endings_test.dart +++ b/test/get/preserve_lock_file_line_endings_test.dart @@ -3,7 +3,7 @@ // BSD-style license that can be found in the LICENSE file. import 'package:path/path.dart' as path; -import 'package:pub/src/entrypoint.dart'; +import 'package:pub/src/lock_file.dart'; import 'package:test/test.dart'; import '../descriptor.dart' as d; diff --git a/test/hosted/short_syntax_test.dart b/test/hosted/short_syntax_test.dart index 5d4cf281f..da6d75434 100644 --- a/test/hosted/short_syntax_test.dart +++ b/test/hosted/short_syntax_test.dart @@ -45,6 +45,7 @@ void main() { 'description': { 'name': 'foo', 'url': globalServer.url, + 'sha256': matches(RegExp(r'[0-9a-f]{64}')) }, 'version': '1.2.3', }); diff --git a/test/lock_file_test.dart b/test/lock_file_test.dart index 1bfac5e0f..20e397686 100644 --- a/test/lock_file_test.dart +++ b/test/lock_file_test.dart @@ -222,21 +222,27 @@ packages: test('serialize() dumps the lockfile to YAML', () { var lockfile = LockFile([ PackageId( - 'foo', - Version.parse('1.2.3'), - ResolvedHostedDescription( - HostedDescription('foo', 'https://foo.com'))), + 'foo', + Version.parse('1.2.3'), + ResolvedHostedDescription( + HostedDescription('foo', 'https://foo.com'), + sha256: null, + ), + ), PackageId( - 'bar', - Version.parse('3.2.1'), - ResolvedHostedDescription( - HostedDescription('bar', 'https://bar.com'))), + 'bar', + Version.parse('3.2.1'), + ResolvedHostedDescription( + HostedDescription('bar', 'https://bar.com'), + sha256: null, + ), + ), ], devDependencies: { 'bar' }); expect( - loadYaml(lockfile.serialize('')), + loadYaml(lockfile.serialize('', cache)), equals({ 'sdks': {'dart': 'any'}, 'packages': { diff --git a/test/package_server.dart b/test/package_server.dart index 6aaea6893..26d773091 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -6,8 +6,10 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:crypto/crypto.dart'; import 'package:path/path.dart' as p; import 'package:pub/src/third_party/tar/tar.dart'; +import 'package:pub/src/utils.dart' show hexEncode; import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:shelf/shelf_io.dart' as shelf_io; @@ -24,9 +26,12 @@ class PackageServer { /// Handlers of requests. Last matching handler will be used. final List<_PatternAndHandler> _handlers = []; - // A list of all the requests recieved up till now. + // A list of all the requests received up till now. final List requestedPaths = []; + // Setting this to true will make automatic calculation of content-hashes. + bool serveContentHashes = false; + PackageServer._(this._inner) { _inner.mount((request) { final path = request.url.path; @@ -53,7 +58,7 @@ class PackageServer { PackageServer._(await shelf_io.IOServer.bind('localhost', 0)); server.handle( _versionInfoPattern, - (shelf.Request request) { + (shelf.Request request) async { final parts = request.url.pathSegments; assert(parts[0] == 'api'); assert(parts[1] == 'packages'); @@ -66,13 +71,20 @@ class PackageServer { return shelf.Response.ok(jsonEncode({ 'name': name, 'uploaders': ['nweiz@google.com'], - 'versions': package.versions.values - .map((version) => packageVersionApiMap( - server._inner.url.toString(), - version.pubspec, - retracted: version.isRetracted, - )) - .toList(), + 'versions': [ + for (final version in package.versions.values) + { + 'pubspec': version.pubspec, + 'version': version.version.toString(), + 'archive_url': + '${server.url}/packages/$name/versions/${version.version}.tar.gz', + if (version.isRetracted) 'retracted': true, + if (version.sha256 != null || server.serveContentHashes) + 'archive_sha256': version.sha256 ?? + hexEncode( + (await sha256.bind(version.contents()).first).bytes) + } + ], if (package.isDiscontinued) 'isDiscontinued': true, if (package.discontinuedReplacementText != null) 'replacedBy': package.discontinuedReplacementText, @@ -207,7 +219,7 @@ class PackageServer { // file mode mode: 420, // size: 100, - modified: DateTime.now(), + modified: DateTime.fromMicrosecondsSinceEpoch(0), userName: 'pub', groupName: 'pub', ), @@ -243,6 +255,16 @@ class PackageServer { void retractPackageVersion(String name, String version) { _packages[name]!.versions[version]!.isRetracted = true; } + + /// Useful for testing handling of a wrong hash. + void setSha256(String name, String version, String sha256) { + _packages[name]!.versions[version]!.sha256 = sha256; + } + + Future getSha256(String name, String version) async { + final v = _packages[name]!.versions[version]!; + return v.sha256 ?? hexEncode((await sha256.bind(v.contents()).first).bytes); + } } class _ServedPackage { @@ -256,6 +278,8 @@ class _ServedPackageVersion { final Map pubspec; final Stream> Function() contents; bool isRetracted = false; + // Overrides the calculated sha256. + String? sha256; Version get version => Version.parse(pubspec['version']); diff --git a/test/pubspec_test.dart b/test/pubspec_test.dart index 1e9c8bd2f..350a72bdb 100644 --- a/test/pubspec_test.dart +++ b/test/pubspec_test.dart @@ -291,8 +291,10 @@ dependencies: expect(foo.name, equals('foo')); expect(foo.source.name, 'hosted'); expect( - ResolvedHostedDescription(foo.description as HostedDescription) - .serializeForLockfile(containingDir: null), + ResolvedHostedDescription( + foo.description as HostedDescription, + sha256: null, + ).serializeForLockfile(containingDir: null), { 'url': 'https://example.org/pub/', 'name': 'bar', @@ -317,8 +319,10 @@ dependencies: expect(foo.name, equals('foo')); expect(foo.source.name, 'hosted'); expect( - ResolvedHostedDescription(foo.description as HostedDescription) - .serializeForLockfile(containingDir: null), + ResolvedHostedDescription( + foo.description as HostedDescription, + sha256: null, + ).serializeForLockfile(containingDir: null), { 'url': 'https://example.org/pub/', 'name': 'foo', @@ -342,8 +346,10 @@ dependencies: expect(foo.name, equals('foo')); expect(foo.source.name, 'hosted'); expect( - ResolvedHostedDescription(foo.description as HostedDescription) - .serializeForLockfile(containingDir: null), + ResolvedHostedDescription( + foo.description as HostedDescription, + sha256: null, + ).serializeForLockfile(containingDir: null), { 'url': 'https://example.org/pub/', 'name': 'foo', @@ -367,8 +373,10 @@ dependencies: expect(foo.name, equals('foo')); expect(foo.source.name, 'hosted'); expect( - ResolvedHostedDescription(foo.description as HostedDescription) - .serializeForLockfile(containingDir: null), + ResolvedHostedDescription( + foo.description as HostedDescription, + sha256: null, + ).serializeForLockfile(containingDir: null), { 'url': 'https://pub.dartlang.org', 'name': 'bar', @@ -411,8 +419,10 @@ dependencies: expect(foo.name, equals('foo')); expect(foo.source.name, 'hosted'); expect( - ResolvedHostedDescription(foo.description as HostedDescription) - .serializeForLockfile(containingDir: null), + ResolvedHostedDescription( + foo.description as HostedDescription, + sha256: null, + ).serializeForLockfile(containingDir: null), { 'url': 'https://pub.dartlang.org', 'name': 'foo', diff --git a/test/reformat_ranges_test.dart b/test/reformat_ranges_test.dart index 9a88df6b5..c1ff8f0d1 100644 --- a/test/reformat_ranges_test.dart +++ b/test/reformat_ranges_test.dart @@ -12,6 +12,7 @@ import 'package:test/test.dart'; void main() { final description = ResolvedHostedDescription( HostedDescription('foo', 'https://pub.dartlang.org'), + sha256: null, ); test('reformatMax when max has a build identifier', () { expect( diff --git a/test/test_pub.dart b/test/test_pub.dart index 3c2f10222..99f03f44b 100644 --- a/test/test_pub.dart +++ b/test/test_pub.dart @@ -24,6 +24,7 @@ import 'package:pub/src/io.dart'; import 'package:pub/src/lock_file.dart'; import 'package:pub/src/log.dart' as log; import 'package:pub/src/package_name.dart'; +import 'package:pub/src/source/hosted.dart'; import 'package:pub/src/system_cache.dart'; import 'package:pub/src/utils.dart'; import 'package:pub/src/validator.dart'; @@ -624,7 +625,10 @@ Future createLockFile(String package, _createLockFile(cache, sandbox: dependenciesInSandBox, hosted: hosted); await d.dir(package, [ - d.file('pubspec.lock', lockFile.serialize(p.join(d.sandbox, package))), + d.file( + 'pubspec.lock', + lockFile.serialize(p.join(d.sandbox, package), cache), + ), d.file( '.packages', lockFile.packagesFile( @@ -680,7 +684,17 @@ LockFile _createLockFile(SystemCache cache, containingDir: p.join(d.sandbox, appPath))), if (hosted != null) ...hosted.entries.map( - (entry) => cache.hosted.idFor(entry.key, Version.parse(entry.value))) + (entry) => PackageId( + entry.key, + Version.parse(entry.value), + ResolvedHostedDescription( + HostedDescription( + entry.key, + 'https://pub.dev', + ), + sha256: null), + ), + ) ]; return LockFile(packages); @@ -720,38 +734,6 @@ Map packageMap( return package; } -/// Returns a Map in the format used by the pub.dartlang.org API to represent a -/// package version. -/// -/// [pubspec] is the parsed pubspec of the package version. If [full] is true, -/// this returns the complete map, including metadata that's only included when -/// requesting the package version directly. -Map packageVersionApiMap(String hostedUrl, Map pubspec, - {bool retracted = false, bool full = false}) { - var name = pubspec['name']; - var version = pubspec['version']; - var map = { - 'pubspec': pubspec, - 'version': version, - 'archive_url': '$hostedUrl/packages/$name/versions/$version.tar.gz', - }; - - if (retracted) { - map['retracted'] = true; - } - - if (full) { - map.addAll({ - 'downloads': 0, - 'created': '2012-09-25T18:38:28.685260', - 'libraries': ['$name.dart'], - 'uploader': ['nweiz@google.com'] - }); - } - - return map; -} - /// Returns the name of the shell script for a binstub named [name]. /// /// Adds a ".bat" extension on Windows. diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt index 7b01ac06f..c1d27de40 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt @@ -10,6 +10,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -30,7 +31,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } } @@ -51,7 +53,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } }, "latest": "2.2.3", @@ -66,7 +69,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4" } }, "constraintBumped": "^2.2.3", @@ -78,7 +82,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } }, @@ -90,7 +95,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } }, "constraintBumped": null, @@ -110,7 +116,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4" } }, "constraintBumped": "^2.2.3", @@ -122,7 +129,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } }, @@ -134,7 +142,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } }, "constraintBumped": null, @@ -167,6 +176,7 @@ packages: dependency: "direct main" description: name: foo + sha256: bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4 url: "http://localhost:$PORT" source: hosted version: "2.2.3" @@ -174,6 +184,7 @@ packages: dependency: transitive description: name: transitive + sha256: d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1 url: "http://localhost:$PORT" source: hosted version: "1.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt index b4726c5b3..6d4d76b79 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt @@ -10,6 +10,7 @@ packages: dependency: "direct main" description: name: bar + sha256: "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -17,6 +18,7 @@ packages: dependency: "direct main" description: name: boo + sha256: "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -24,6 +26,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -44,7 +47,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" } } }, @@ -57,7 +61,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" } } }, @@ -70,7 +75,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } } @@ -91,7 +97,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" } }, "latest": "2.2.3", @@ -106,7 +113,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "711bd294f88edb3c954efe18925449490aabfe36ab8bc95ec092a963c509f801" } }, "constraintBumped": "^2.2.3", @@ -118,7 +126,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" } } } @@ -132,7 +141,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "711bd294f88edb3c954efe18925449490aabfe36ab8bc95ec092a963c509f801" } }, "constraintBumped": "^2.2.3", @@ -144,7 +154,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" } } } @@ -158,7 +169,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" } }, "latest": "1.2.4", @@ -172,7 +184,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" } }, "constraintBumped": "^1.0.0", @@ -184,7 +197,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" } } } @@ -198,7 +212,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" } }, "constraintBumped": "^1.2.4", @@ -210,7 +225,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" } } } @@ -224,7 +240,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" } }, "constraintBumped": "^1.2.4", @@ -236,7 +253,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "boo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" } } } @@ -250,7 +268,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } }, "latest": "2.2.3", @@ -264,7 +283,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "807fa476dce54e18976270a25ed38efcb4925349f9ddf3699b09e51c21ecddc6" } }, "constraintBumped": "^1.0.0", @@ -276,7 +296,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } } @@ -290,7 +311,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -302,7 +324,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } } @@ -316,7 +339,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -328,7 +352,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" } } } @@ -355,6 +380,7 @@ packages: dependency: "direct main" description: name: bar + sha256: "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -362,6 +388,7 @@ packages: dependency: "direct main" description: name: boo + sha256: "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -369,6 +396,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "807fa476dce54e18976270a25ed38efcb4925349f9ddf3699b09e51c21ecddc6" url: "http://localhost:$PORT" source: hosted version: "1.2.4" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt b/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt index ca80a34d8..b7fc7a50a 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt @@ -34,7 +34,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } }, @@ -47,7 +48,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } } } @@ -86,7 +88,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -98,7 +101,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } } @@ -112,7 +116,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } }, "latest": "2.2.3", @@ -127,7 +132,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -139,7 +145,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } } @@ -153,7 +160,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -165,7 +173,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } } @@ -179,7 +188,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } }, "latest": "1.0.0", diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt index 69a06d4c4..15648b17e 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt @@ -17,6 +17,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -50,7 +51,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -88,7 +90,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } }, "latest": "2.0.0", @@ -115,7 +118,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -141,7 +145,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -175,6 +180,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "785f9e487af494545c823fd7bf0576c76eff1ec96ee00923038e9b13395f8886" url: "http://localhost:$PORT" source: hosted version: "2.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt index 50ee46fbe..7385131d2 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt @@ -10,6 +10,7 @@ packages: dependency: "direct main" description: name: foo + sha256: d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -17,6 +18,7 @@ packages: dependency: transitive description: name: transitive + sha256: d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1 url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -37,7 +39,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } }, @@ -50,7 +53,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } } } @@ -71,7 +75,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } }, "latest": "2.2.3", @@ -86,7 +91,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -98,7 +104,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } }, @@ -115,7 +122,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } } } @@ -129,7 +137,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" } }, "constraintBumped": "^2.2.3", @@ -141,7 +150,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" } } }, @@ -158,7 +168,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } } } @@ -172,7 +183,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "transitive", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" } }, "latest": "1.0.0", @@ -202,6 +214,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt index da9222d6c..720b15050 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt @@ -10,6 +10,7 @@ packages: dependency: "direct main" description: name: bar + sha256: ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34 url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -17,6 +18,7 @@ packages: dependency: "direct main" description: name: baz + sha256: "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -24,6 +26,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -44,7 +47,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" } } }, @@ -57,7 +61,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" } } }, @@ -70,7 +75,8 @@ $ dependency_services list "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -91,7 +97,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" } }, "latest": "2.0.0", @@ -107,7 +114,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb" } }, "constraintBumped": "^2.0.0", @@ -119,7 +127,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" } } }, @@ -131,7 +140,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb" } }, "constraintBumped": "^3.0.1", @@ -143,7 +153,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -157,7 +168,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" } }, "latest": "1.1.0", @@ -172,7 +184,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "7f4b0233fe5eae65af207f346e532c13c1a39fd8442fb64a02e83b73e4dcadb5" } }, "constraintBumped": "^1.1.0", @@ -184,7 +197,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" } } } @@ -198,7 +212,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "7f4b0233fe5eae65af207f346e532c13c1a39fd8442fb64a02e83b73e4dcadb5" } }, "constraintBumped": "^1.1.0", @@ -210,7 +225,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "baz", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" } } } @@ -224,7 +240,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } }, "latest": "3.0.1", @@ -238,7 +255,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "bfb787d23489e4111a8f4a0940fd43169c49f6313c0c231e0ba73bc5e31d6704" } }, "constraintBumped": "^1.0.0", @@ -250,7 +268,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -264,7 +283,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "785f9e487af494545c823fd7bf0576c76eff1ec96ee00923038e9b13395f8886" } }, "constraintBumped": "^2.0.0", @@ -276,7 +296,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } } @@ -290,7 +311,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb" } }, "constraintBumped": "^3.0.1", @@ -302,7 +324,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" } } }, @@ -314,7 +337,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb" } }, "constraintBumped": "^2.0.0", @@ -326,7 +350,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "bar", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" } } } @@ -353,6 +378,7 @@ packages: dependency: "direct main" description: name: bar + sha256: f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb url: "http://localhost:$PORT" source: hosted version: "2.0.0" @@ -360,6 +386,7 @@ packages: dependency: "direct main" description: name: baz + sha256: "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -367,6 +394,7 @@ packages: dependency: "direct main" description: name: foo + sha256: a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb url: "http://localhost:$PORT" source: hosted version: "3.0.1" diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index fc69b0399..bd1971009 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -56,6 +56,9 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | x-content-type-options: nosniff [E] | server: dart:io with Shelf [E] IO : Creating $FILE from stream +[E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +[E] FINE: Contents: +[E] | 5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72 [E] FINE: Created $FILE from stream [E] IO : Created temp directory $DIR [E] IO : Reading binary file $FILE. @@ -76,6 +79,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | dependency: "direct main" [E] | description: [E] | name: foo +[E] | sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" [E] | url: "http://localhost:$PORT" [E] | source: hosted [E] | version: "1.0.0" @@ -133,6 +137,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -194,6 +199,9 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/ | x-content-type-options: nosniff | server: dart:io with Shelf IO : Creating $FILE from stream +IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +FINE: Contents: + | 5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72 FINE: Created $FILE from stream IO : Created temp directory $DIR IO : Reading binary file $FILE. @@ -214,6 +222,7 @@ FINE: Contents: | dependency: "direct main" | description: | name: foo + | sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" | url: "http://localhost:$PORT" | source: hosted | version: "1.0.0" diff --git a/test/testdata/goldens/help_test/pub get --help.txt b/test/testdata/goldens/help_test/pub get --help.txt index 104f69cd5..3a34c5d29 100644 --- a/test/testdata/goldens/help_test/pub get --help.txt +++ b/test/testdata/goldens/help_test/pub get --help.txt @@ -10,6 +10,7 @@ Usage: pub get [arguments...] network. -n, --dry-run Report what dependencies would change but don't change any. + --enforce-lockfile Only use resolution in existing lockfile. --[no-]precompile Build executables in immediate dependencies. --legacy-packages-file Generate the legacy ".packages" file -C, --directory= Run this in the directory. diff --git a/test/utils_test.dart b/test/utils_test.dart index 78ccbafbc..cc51066fc 100644 --- a/test/utils_test.dart +++ b/test/utils_test.dart @@ -162,4 +162,18 @@ b: {}''')); } }); }); + + test('hexEncode', () { + expect(hexEncode([]), ''); + expect(hexEncode([255, 0, 1, 240]), 'ff0001f0'); + expect(() => hexEncode([256, 0, 1]), throwsA(isA())); + }); + test('hexDecode', () { + expect(hexDecode(''), []); + expect(hexDecode('ff0001f0abcdef'), [255, 0, 1, 240, 171, 205, 239]); + expect(hexDecode('FF0001F0ABCDEF'), [255, 0, 1, 240, 171, 205, 239]); + expect(() => hexDecode('F'), throwsA(isA())); + expect(() => hexDecode('0p'), throwsA(isA())); + expect(() => hexDecode('p0'), throwsA(isA())); + }); } From 628440dc01dcfb39feb9447e25200e5e19b5eba5 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 8 Jul 2022 10:19:54 +0000 Subject: [PATCH 02/51] Update repo spec --- doc/repository-spec-v2.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/repository-spec-v2.md b/doc/repository-spec-v2.md index 224aa2db7..57398472f 100644 --- a/doc/repository-spec-v2.md +++ b/doc/repository-spec-v2.md @@ -120,7 +120,7 @@ parse the ``. The `dart pub` client allows users to save an opaque `` for each ``. When the `dart pub` client makes a request to a `` for which it has a `` stored, it will attach an `Authorization` header -as follows: +as follows: * `Authorization: Bearer ` @@ -229,6 +229,7 @@ server, this could work in many different ways. "version": "", "retracted": true || false, /* optional field, false if omitted */ "archive_url": "https://.../archive.tar.gz", + "archive_sha256": "95cbaad58e2cf32d1aa852f20af1fcda1820ead92a4b1447ea7ba1ba18195d27" "pubspec": { /* pubspec contents as JSON object */ } @@ -238,6 +239,7 @@ server, this could work in many different ways. "version": "", "retracted": true || false, /* optional field, false if omitted */ "archive_url": "https://.../archive.tar.gz", + "archive_sha256": "95cbaad58e2cf32d1aa852f20af1fcda1820ead92a4b1447ea7ba1ba18195d27" "pubspec": { /* pubspec contents as JSON object */ } @@ -256,6 +258,15 @@ parameters. This allows for the server to return signed-URLs for S3, GCS or other blob storage service. If temporary URLs are returned it is wise to not set expiration to less than 25 minutes (to allow for retries and clock drift). +The `archive_sha256` should be the hex-encoded sha256 checksum of the file are +archive_url. It is an optional field that allows the pub client to verify the +integrity of downloaded archive. + +The `archive_sha256` also provides an easy way for clients to detect if +something has changed on the server. In the absense of this field the client can +still download the archive to obtain a checksum and detect changes to the +archive. + If `` for the server returning `archive_url` is a prefix of `archive_url`, then the `Authorization: Bearer ` is also included when `archive_url` is requested. Example: if `https://pub.example.com/path` returns From 6308de721244abbe00a3918e199f652cf4943603 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 8 Jul 2022 10:40:14 +0000 Subject: [PATCH 03/51] Remove os-marker from gzip output (testing only) --- test/package_server.dart | 22 +++++++- .../Adding transitive.txt | 22 ++++---- .../dependency_services_test/Compatible.txt | 56 +++++++++---------- .../No pubspec.lock.txt | 20 +++---- .../Relative paths are allowed.txt | 12 ++-- .../Removing transitive.txt | 26 ++++----- .../multibreaking.txt | 56 +++++++++---------- ...--verbose and on unexpected exceptions.txt | 10 ++-- 8 files changed, 121 insertions(+), 103 deletions(-) diff --git a/test/package_server.dart b/test/package_server.dart index 26d773091..b2d42bcbf 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -68,6 +68,7 @@ class PackageServer { if (package == null) { return shelf.Response.notFound('No package named $name'); } + return shelf.Response.ok(jsonEncode({ 'name': name, 'uploaders': ['nweiz@google.com'], @@ -232,13 +233,30 @@ class PackageServer { for (final e in contents ?? []) { addDescriptor(e, ''); } - return Stream.fromIterable(entries) + return replaceOs(Stream.fromIterable(entries) .transform(tarWriterWith(format: OutputFormat.gnuLongName)) - .transform(gzip.encoder); + .transform(gzip.encoder)); }, ); } + /// Replaces the 9th entry in [stream] with a 0. This replaces the os entry + /// of a gzip stream, giving us the same stream on all platforms. + Stream> replaceOs(Stream> stream) async* { + var i = 0; + await for (final t in stream) { + if (i > 9 || (i + t.length < 9)) { + yield t; + i += t.length; + continue; + } + yield t.sublist(0, 9 - i); + yield [0]; + yield t.sublist(9 - i + 1); + i += t.length; + } + } + // Mark a package discontinued. void discontinue(String name, {bool isDiscontinued = true, String? replacementText}) { diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt index c1d27de40..3b16db157 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt @@ -10,7 +10,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + sha256: "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -32,7 +32,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } } @@ -54,7 +54,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } }, "latest": "2.2.3", @@ -70,7 +70,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4" + "sha256": "fc06d01652f7b73f789abeb5b61aeb68b13cd472f87610cb8fb80e402a9139ff" } }, "constraintBumped": "^2.2.3", @@ -83,7 +83,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } }, @@ -96,7 +96,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } }, "constraintBumped": null, @@ -117,7 +117,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4" + "sha256": "fc06d01652f7b73f789abeb5b61aeb68b13cd472f87610cb8fb80e402a9139ff" } }, "constraintBumped": "^2.2.3", @@ -130,7 +130,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } }, @@ -143,7 +143,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } }, "constraintBumped": null, @@ -176,7 +176,7 @@ packages: dependency: "direct main" description: name: foo - sha256: bee723e4e4676034abb66fd10ffe1ce600ff0769c2ca2c8092bcb4732d930dd4 + sha256: fc06d01652f7b73f789abeb5b61aeb68b13cd472f87610cb8fb80e402a9139ff url: "http://localhost:$PORT" source: hosted version: "2.2.3" @@ -184,7 +184,7 @@ packages: dependency: transitive description: name: transitive - sha256: d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1 + sha256: "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" url: "http://localhost:$PORT" source: hosted version: "1.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt index 6d4d76b79..4fbaffbd0 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt @@ -10,7 +10,7 @@ packages: dependency: "direct main" description: name: bar - sha256: "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + sha256: ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20 url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -18,7 +18,7 @@ packages: dependency: "direct main" description: name: boo - sha256: "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + sha256: "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -26,7 +26,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + sha256: "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -48,7 +48,7 @@ $ dependency_services list "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + "sha256": "ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20" } } }, @@ -62,7 +62,7 @@ $ dependency_services list "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + "sha256": "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" } } }, @@ -76,7 +76,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } } @@ -98,7 +98,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + "sha256": "ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20" } }, "latest": "2.2.3", @@ -114,7 +114,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "711bd294f88edb3c954efe18925449490aabfe36ab8bc95ec092a963c509f801" + "sha256": "adcfe9ac3d6955fd4332f29f47bf3e814e388e2da7c2bc55d4561971bf8b5335" } }, "constraintBumped": "^2.2.3", @@ -127,7 +127,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + "sha256": "ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20" } } } @@ -142,7 +142,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "711bd294f88edb3c954efe18925449490aabfe36ab8bc95ec092a963c509f801" + "sha256": "adcfe9ac3d6955fd4332f29f47bf3e814e388e2da7c2bc55d4561971bf8b5335" } }, "constraintBumped": "^2.2.3", @@ -155,7 +155,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + "sha256": "ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20" } } } @@ -170,7 +170,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + "sha256": "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" } }, "latest": "1.2.4", @@ -185,7 +185,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" } }, "constraintBumped": "^1.0.0", @@ -198,7 +198,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + "sha256": "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" } } } @@ -213,7 +213,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" } }, "constraintBumped": "^1.2.4", @@ -226,7 +226,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + "sha256": "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" } } } @@ -241,7 +241,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "3eed60ebb7f44cbc329bf076f671a4396993ef169273342858b797e7b7389e4d" + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" } }, "constraintBumped": "^1.2.4", @@ -254,7 +254,7 @@ $ dependency_services report "description": { "name": "boo", "url": "http://localhost:$PORT", - "sha256": "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + "sha256": "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" } } } @@ -269,7 +269,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } }, "latest": "2.2.3", @@ -284,7 +284,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "807fa476dce54e18976270a25ed38efcb4925349f9ddf3699b09e51c21ecddc6" + "sha256": "88f2f9251967bf04bd478873f074b9d8df9f1c959afc150ba3b0ea813d48161e" } }, "constraintBumped": "^1.0.0", @@ -297,7 +297,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } } @@ -312,7 +312,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -325,7 +325,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } } @@ -340,7 +340,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -353,7 +353,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "18c521c0693bf9b3cff95e607df6350daeb1eb94dcb127e8724ef04b5cbb6df3" + "sha256": "1614d63c0867d0994f75a231be7ee394a4f30cdeede4c7ea471fcad354c23d1f" } } } @@ -380,7 +380,7 @@ packages: dependency: "direct main" description: name: bar - sha256: "048ba0044428b200d7f0992e4cd21774ffe1e1e5f68cd929e57ca00bd74ebe33" + sha256: ea004e8b0069df9e9827b101b64aaad455cc358849f1801dc48a41111cabbe20 url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -388,7 +388,7 @@ packages: dependency: "direct main" description: name: boo - sha256: "56a315b93e63113914b72aae210ed71418a418be2e95226b0290ff6f3d2edf3f" + sha256: "7971e197614f18130070007a54f446366c6e594f0ed159ae2c4e2b42972c426b" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -396,7 +396,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "807fa476dce54e18976270a25ed38efcb4925349f9ddf3699b09e51c21ecddc6" + sha256: "88f2f9251967bf04bd478873f074b9d8df9f1c959afc150ba3b0ea813d48161e" url: "http://localhost:$PORT" source: hosted version: "1.2.4" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt b/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt index b7fc7a50a..8bf69718e 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/No pubspec.lock.txt @@ -35,7 +35,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } }, @@ -49,7 +49,7 @@ $ dependency_services list "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } } } @@ -89,7 +89,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -102,7 +102,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } } @@ -117,7 +117,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } }, "latest": "2.2.3", @@ -133,7 +133,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -146,7 +146,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } } @@ -161,7 +161,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -174,7 +174,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } } @@ -189,7 +189,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } }, "latest": "1.0.0", diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt index 15648b17e..82badd7c1 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt @@ -17,7 +17,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + sha256: "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -52,7 +52,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -91,7 +91,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } }, "latest": "2.0.0", @@ -119,7 +119,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -146,7 +146,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -180,7 +180,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "785f9e487af494545c823fd7bf0576c76eff1ec96ee00923038e9b13395f8886" + sha256: c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0 url: "http://localhost:$PORT" source: hosted version: "2.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt index 7385131d2..04986112b 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt @@ -10,7 +10,7 @@ packages: dependency: "direct main" description: name: foo - sha256: d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd + sha256: "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" url: "http://localhost:$PORT" source: hosted version: "1.2.3" @@ -18,7 +18,7 @@ packages: dependency: transitive description: name: transitive - sha256: d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1 + sha256: "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -40,7 +40,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } }, @@ -54,7 +54,7 @@ $ dependency_services list "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } } } @@ -76,7 +76,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } }, "latest": "2.2.3", @@ -92,7 +92,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -105,7 +105,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } }, @@ -123,7 +123,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } } } @@ -138,7 +138,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" } }, "constraintBumped": "^2.2.3", @@ -151,7 +151,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "d13ce27ca2161ad3684af5fb44c823fc0890213e57b9e1eb9edc7252be7a35dd" + "sha256": "72f6a04c4af0d78e4f1a1e2eb00a850843e6c0c5233ac2ca911aa061cbd5f8f1" } } }, @@ -169,7 +169,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } } } @@ -184,7 +184,7 @@ $ dependency_services report "description": { "name": "transitive", "url": "http://localhost:$PORT", - "sha256": "d8731eb2273b5ceef9c3ce3bbd2393801e6848503651d5d12ec05d803a0959d1" + "sha256": "8d245de5cde3ab3293e4cdea516c6a0395e24d338688279bab5f6c97bffa0915" } }, "latest": "1.0.0", @@ -214,7 +214,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "0f9c20f8f3233b848d963ba59a227f4b87ce6155a40dec19f88275696bc3bff3" + sha256: bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt index 720b15050..3767075d5 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt @@ -10,7 +10,7 @@ packages: dependency: "direct main" description: name: bar - sha256: ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34 + sha256: "4de00552ae3719481f5f0e30b82ecb8b14a62907553b217e7ca178e80625329a" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -18,7 +18,7 @@ packages: dependency: "direct main" description: name: baz - sha256: "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + sha256: "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -26,7 +26,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + sha256: "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -48,7 +48,7 @@ $ dependency_services list "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" + "sha256": "4de00552ae3719481f5f0e30b82ecb8b14a62907553b217e7ca178e80625329a" } } }, @@ -62,7 +62,7 @@ $ dependency_services list "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + "sha256": "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" } } }, @@ -76,7 +76,7 @@ $ dependency_services list "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -98,7 +98,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" + "sha256": "4de00552ae3719481f5f0e30b82ecb8b14a62907553b217e7ca178e80625329a" } }, "latest": "2.0.0", @@ -115,7 +115,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb" + "sha256": "b8187621010649d6385788d7630adcd88d6548a7938899b6f18820961df3b879" } }, "constraintBumped": "^2.0.0", @@ -128,7 +128,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" + "sha256": "4de00552ae3719481f5f0e30b82ecb8b14a62907553b217e7ca178e80625329a" } } }, @@ -141,7 +141,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb" + "sha256": "2347a7792f73d0f8cc8aa41d4895317bd1745724b8bc77d8c03faf821c9059b7" } }, "constraintBumped": "^3.0.1", @@ -154,7 +154,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -169,7 +169,7 @@ $ dependency_services report "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + "sha256": "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" } }, "latest": "1.1.0", @@ -185,7 +185,7 @@ $ dependency_services report "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "7f4b0233fe5eae65af207f346e532c13c1a39fd8442fb64a02e83b73e4dcadb5" + "sha256": "7474da026b513eafecba9d1c79a8a3b4a9ef5158730e0968383063b3237c5dec" } }, "constraintBumped": "^1.1.0", @@ -198,7 +198,7 @@ $ dependency_services report "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + "sha256": "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" } } } @@ -213,7 +213,7 @@ $ dependency_services report "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "7f4b0233fe5eae65af207f346e532c13c1a39fd8442fb64a02e83b73e4dcadb5" + "sha256": "7474da026b513eafecba9d1c79a8a3b4a9ef5158730e0968383063b3237c5dec" } }, "constraintBumped": "^1.1.0", @@ -226,7 +226,7 @@ $ dependency_services report "description": { "name": "baz", "url": "http://localhost:$PORT", - "sha256": "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + "sha256": "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" } } } @@ -241,7 +241,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } }, "latest": "3.0.1", @@ -256,7 +256,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "bfb787d23489e4111a8f4a0940fd43169c49f6313c0c231e0ba73bc5e31d6704" + "sha256": "efa386ac7cc7698525e2e820a90e6bcee5d6c071de4315051a0fb2f3aff5d084" } }, "constraintBumped": "^1.0.0", @@ -269,7 +269,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -284,7 +284,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "785f9e487af494545c823fd7bf0576c76eff1ec96ee00923038e9b13395f8886" + "sha256": "c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0" } }, "constraintBumped": "^2.0.0", @@ -297,7 +297,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } } @@ -312,7 +312,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb" + "sha256": "2347a7792f73d0f8cc8aa41d4895317bd1745724b8bc77d8c03faf821c9059b7" } }, "constraintBumped": "^3.0.1", @@ -325,7 +325,7 @@ $ dependency_services report "description": { "name": "foo", "url": "http://localhost:$PORT", - "sha256": "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + "sha256": "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" } } }, @@ -338,7 +338,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb" + "sha256": "b8187621010649d6385788d7630adcd88d6548a7938899b6f18820961df3b879" } }, "constraintBumped": "^2.0.0", @@ -351,7 +351,7 @@ $ dependency_services report "description": { "name": "bar", "url": "http://localhost:$PORT", - "sha256": "ae5907b5f0861b6000253fa396ab209effe18146c25e025c283bb61c0f4fbe34" + "sha256": "4de00552ae3719481f5f0e30b82ecb8b14a62907553b217e7ca178e80625329a" } } } @@ -378,7 +378,7 @@ packages: dependency: "direct main" description: name: bar - sha256: f8b42e4481e9043e6a3c23e02c4d3f6c0d5c3b3a250d2630cfbda7655acfc6bb + sha256: b8187621010649d6385788d7630adcd88d6548a7938899b6f18820961df3b879 url: "http://localhost:$PORT" source: hosted version: "2.0.0" @@ -386,7 +386,7 @@ packages: dependency: "direct main" description: name: baz - sha256: "28c999b54acd2f19431e094b1dbc6ada555d3badf55abcbbd5e0048d574fedb2" + sha256: "377433f0e0aff092191e57de97f5869cad0dd0779ee6d31e7096b84878ca41e8" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -394,7 +394,7 @@ packages: dependency: "direct main" description: name: foo - sha256: a5d5d4d235933e8327fc980fbd58ed6a41aa030fc25898963befb1ed24b01ffb + sha256: "2347a7792f73d0f8cc8aa41d4895317bd1745724b8bc77d8c03faf821c9059b7" url: "http://localhost:$PORT" source: hosted version: "3.0.1" diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index bd1971009..bab4187f1 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -58,7 +58,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] IO : Creating $FILE from stream [E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. [E] FINE: Contents: -[E] | 5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72 +[E] | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb [E] FINE: Created $FILE from stream [E] IO : Created temp directory $DIR [E] IO : Reading binary file $FILE. @@ -79,7 +79,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | dependency: "direct main" [E] | description: [E] | name: foo -[E] | sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" +[E] | sha256: "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" [E] | url: "http://localhost:$PORT" [E] | source: hosted [E] | version: "1.0.0" @@ -137,7 +137,7 @@ packages: dependency: "direct main" description: name: foo - sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + sha256: "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" url: "http://localhost:$PORT" source: hosted version: "1.0.0" @@ -201,7 +201,7 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/ IO : Creating $FILE from stream IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. FINE: Contents: - | 5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72 + | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb FINE: Created $FILE from stream IO : Created temp directory $DIR IO : Reading binary file $FILE. @@ -222,7 +222,7 @@ FINE: Contents: | dependency: "direct main" | description: | name: foo - | sha256: "5b4d3c47282cbb9f09f0c6a3570fe5dd1db39db04d6136ef7594fb2eeef8df72" + | sha256: "439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb" | url: "http://localhost:$PORT" | source: hosted | version: "1.0.0" From 5b9cc9105238daad957af18b27fadb285f09f41d Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 14:58:25 +0200 Subject: [PATCH 04/51] Update lib/src/command/get.dart Co-authored-by: Jonas Finnemann Jensen --- lib/src/command/get.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/command/get.dart b/lib/src/command/get.dart index f6a7b71f9..a73154a4d 100644 --- a/lib/src/command/get.dart +++ b/lib/src/command/get.dart @@ -30,7 +30,7 @@ class GetCommand extends PubCommand { help: "Report what dependencies would change but don't change any."); argParser.addFlag('enforce-lockfile', - negatable: false, help: 'Only use resolution in existing lockfile.'); + negatable: false, help: 'Only use resolution from existing pubspec.lock.'); argParser.addFlag('precompile', help: 'Build executables in immediate dependencies.'); From e2b8cf9213a63fb637b43a126bf338742a6f61fb Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 13:06:17 +0000 Subject: [PATCH 05/51] Use hex from package:convert --- lib/src/utils.dart | 42 +++--------------------------------------- pubspec.yaml | 1 + 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index b2a08dd51..09f38e8bb 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -10,6 +10,7 @@ import 'dart:math' as math; import 'dart:typed_data'; import 'package:collection/collection.dart'; +import 'package:convert/convert.dart'; import 'package:crypto/crypto.dart' as crypto; import 'package:pub_semver/pub_semver.dart'; import 'package:stack_trace/stack_trace.dart'; @@ -330,46 +331,9 @@ String replace(String source, Pattern matcher, String Function(Match) fn) { String sha1(String source) => crypto.sha1.convert(utf8.encode(source)).toString(); -final _hexTable = [ - '0', '1', '2', '3', '4', '5', '6', '7', // - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', -]; - -final _hexTable2 = Map.fromIterables(_hexTable, List.generate(16, (i) => i)); - -String hexEncode(List bytes) { - const mask = (1 << 4) - 1; - final buffer = StringBuffer(); - for (final byte in bytes) { - if (byte > 255 || byte < 0) { - throw FormatException('Bad value in byte list $byte.'); - } - buffer.write(_hexTable[byte >> 4 & mask]); - buffer.write(_hexTable[byte & mask]); - } - return buffer.toString(); -} +String hexEncode(List bytes) => hex.encode(bytes); -Uint8List hexDecode(String string) { - string = string.toLowerCase(); - if (string.length % 2 != 0) { - throw FormatException( - 'Bad hex encoding, must have an even number of characters'); - } - final result = Uint8List(string.length ~/ 2); - for (var i = 0; i < result.length; i++) { - final v = _hexTable2[string[i * 2]]; - if (v == null) { - throw FormatException('Bad char `${string[i * 2]}` in hex encoding'); - } - final v2 = _hexTable2[string[i * 2 + 1]]; - if (v2 == null) { - throw FormatException('Bad char `${string[i * 2 + 1]}` in hex encoding'); - } - result[i] = (v << 4) | v2; - } - return result; -} +Uint8List hexDecode(String string) => hex.decode(string) as Uint8List; /// A regular expression matching a trailing CR character. final _trailingCR = RegExp(r'\r$'); diff --git a/pubspec.yaml b/pubspec.yaml index e44e0e709..4cbbcafae 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -11,6 +11,7 @@ dependencies: async: ^2.6.1 cli_util: ^0.3.5 collection: ^1.15.0 + convert: ^3.0.2 crypto: ^3.0.1 frontend_server_client: ^2.0.0 http: ^0.13.3 From 7f969956200877027cf323a366f0cfcc62f6c985 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 13:14:02 +0000 Subject: [PATCH 06/51] Rename enforceContentHashes -> checkContentHashes --- lib/src/entrypoint.dart | 2 +- lib/src/lock_file.dart | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index fe82d1e6d..b07cdf218 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -348,7 +348,7 @@ class Entrypoint { allowOutdatedHashChecks: !enforceLockfile, ); if (enforceLockfile) { - result.lockFile.enforceContentHashes(cache); + result.lockFile.checkContentHashes(cache); } else { result.lockFile.writeToFile(lockFilePath, cache); } diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 3f0e71ea9..69804f30a 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -313,9 +313,9 @@ Generated by pub on ${DateTime.now()}.'''; return '${JsonEncoder.withIndent(' ').convert(packageConfig.toJson())}\n'; } - /// Throws a [DataException] if the content-hash of some hosted package locked in this lock file differs from the - /// one in the cache. - void enforceContentHashes(SystemCache cache) { + /// Throws a [DataException] if the content-hash of some hosted package locked + /// in this lock file differs from the one in the [cache]. + void checkContentHashes(SystemCache cache) { packages.forEach((name, package) { var description = package.description; if (description is ResolvedHostedDescription) { From 6f6adf0a117fb71a050d343d1cefbb233292c610 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 13:15:46 +0000 Subject: [PATCH 07/51] forEach -> for --- lib/src/lock_file.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 69804f30a..198936c09 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -316,12 +316,12 @@ Generated by pub on ${DateTime.now()}.'''; /// Throws a [DataException] if the content-hash of some hosted package locked /// in this lock file differs from the one in the [cache]. void checkContentHashes(SystemCache cache) { - packages.forEach((name, package) { - var description = package.description; + for (final id in packages.values) { + var description = id.description; if (description is ResolvedHostedDescription) { if (description.sha256 != null) { Uint8List? cachedHash = - description.description.source.sha256FromCache(package, cache); + description.description.source.sha256FromCache(id, cache); if (cachedHash == null) { // This can happen if we resolve from a server not providing hashes, // but we are not downloading the archives. (eg. for a @@ -330,11 +330,11 @@ Generated by pub on ${DateTime.now()}.'''; log.fine('No hash of downloaded archive found'); } else if (!bytesEquals(cachedHash, description.sha256)) { dataError( - 'Cache entry for $name-${package.version} does not have content-hash matching lockfile.'); + 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); } } } - }); + } } /// Returns the serialized YAML text of the lock file. From 5f19d8c21f32e4c852bc657e952c458ef298c89c Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 13:17:34 +0000 Subject: [PATCH 08/51] Don't type local --- lib/src/lock_file.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 198936c09..cd2ea5ead 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -320,7 +320,7 @@ Generated by pub on ${DateTime.now()}.'''; var description = id.description; if (description is ResolvedHostedDescription) { if (description.sha256 != null) { - Uint8List? cachedHash = + final cachedHash = description.description.source.sha256FromCache(id, cache); if (cachedHash == null) { // This can happen if we resolve from a server not providing hashes, From 407c94c77ce22a30dedb28fb9440fb9b13387239 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 13:21:42 +0000 Subject: [PATCH 09/51] forEach -> for --- lib/src/lock_file.dart | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index cd2ea5ead..d554472d1 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -326,8 +326,8 @@ Generated by pub on ${DateTime.now()}.'''; // This can happen if we resolve from a server not providing hashes, // but we are not downloading the archives. (eg. for a // dependency_services run). - // TODO(sigurdm): What should we do here? - log.fine('No hash of downloaded archive found'); + log.fine( + 'No hash of downloaded archive for ${id.name}-${id.version} found'); } else if (!bytesEquals(cachedHash, description.sha256)) { dataError( 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); @@ -345,11 +345,11 @@ Generated by pub on ${DateTime.now()}.'''; String serialize(String? packageDir, SystemCache cache) { // Convert the dependencies to a simple object. var packageMap = {}; - packages.forEach((name, package) { - var description = package.description; + for (final id in packages.values) { + var description = id.description; if (description is ResolvedHostedDescription) { Uint8List? hash = - description.description.source.sha256FromCache(package, cache); + description.description.source.sha256FromCache(id, cache); if (hash == null) { // This can happen if we resolve from a server not providing hashes, // but we are not downloading the archives. (eg. for a @@ -364,19 +364,19 @@ Generated by pub on ${DateTime.now()}.'''; // TODO(Should we really not fail here???). if (!bytesEquals(hash, description.sha256)) { dataError( - 'Cache entry for $name-${package.version} does not have content-hash matching lockfile.'); + 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); } } } - packageMap[name] = { - 'version': package.version.toString(), - 'source': package.source.name, + packageMap[id.name] = { + 'version': id.version.toString(), + 'source': id.source.name, 'description': description.serializeForLockfile(containingDir: packageDir), - 'dependency': _dependencyType(package.name) + 'dependency': _dependencyType(id.name) }; - }); + } var data = { 'sdks': mapMap(sdkConstraints, From 6bf38ae31f1a9e7eeb39e1576f79ebb47fd79b4a Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 5 Aug 2022 14:07:32 +0000 Subject: [PATCH 10/51] Always call LockFile.checkContentHashes - change impossible case to assert --- lib/src/entrypoint.dart | 5 +-- lib/src/global_packages.dart | 1 + lib/src/lock_file.dart | 45 ++++++++++++------- lib/src/source/hosted.dart | 5 +++ test/get/enforce_lockfile_test.dart | 4 +- .../goldens/help_test/pub get --help.txt | 2 +- 6 files changed, 39 insertions(+), 23 deletions(-) diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index b07cdf218..7fb20f0f6 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -347,9 +347,8 @@ class Entrypoint { result.packages, allowOutdatedHashChecks: !enforceLockfile, ); - if (enforceLockfile) { - result.lockFile.checkContentHashes(cache); - } else { + result.lockFile.checkContentHashes(cache); + if (!enforceLockfile) { result.lockFile.writeToFile(lockFilePath, cache); } } diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index 15db8262f..9b272fbe9 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -280,6 +280,7 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam /// Finishes activating package [package] by saving [lockFile] in the cache. void _writeLockFile(String dir, LockFile lockFile, SystemCache cache) { + lockFile.checkContentHashes(cache); writeTextFile(p.join(dir, 'pubspec.lock'), lockFile.serialize(null, cache)); } diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index d554472d1..d314938a1 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -8,6 +8,7 @@ import 'dart:typed_data'; import 'package:collection/collection.dart' hide mapMap; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; +import 'package:pub/src/command_runner.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:source_span/source_span.dart'; import 'package:yaml/yaml.dart'; @@ -319,18 +320,31 @@ Generated by pub on ${DateTime.now()}.'''; for (final id in packages.values) { var description = id.description; if (description is ResolvedHostedDescription) { - if (description.sha256 != null) { - final cachedHash = - description.description.source.sha256FromCache(id, cache); - if (cachedHash == null) { - // This can happen if we resolve from a server not providing hashes, - // but we are not downloading the archives. (eg. for a - // dependency_services run). - log.fine( - 'No hash of downloaded archive for ${id.name}-${id.version} found'); - } else if (!bytesEquals(cachedHash, description.sha256)) { - dataError( - 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); + Uint8List? cachedHash = + description.description.source.sha256FromCache(id, cache); + if (cachedHash == null) { + // This can happen if we resolve from a server not providing hashes, + // but we are not downloading the archives. (eg. for a + // dependency_services run). + log.fine( + 'No hash of downloaded archive for ${id.name}-${id.version} found'); + } else if (description.sha256 == null) { + // We have resolved from a server without archive_sha256 in the + // version listing. Use the sha256 from the archive instead. + description = description.withSha256(cachedHash); + } else { + // TODO(Should we really not fail here???). + if (!bytesEquals(cachedHash, description.sha256)) { + dataError(''' +Cache entry for ${id.name}-${id.version} does not have content-hash matching pubspec.lock. + +This might indicate + +* A corruption of your cache (try running `$topLevelProgram pub cache clean`). +* A corruption of the package repository (the package archive has been changed + on the server). +* The pubspec.lock has been corrupted. +'''); } } } @@ -361,11 +375,8 @@ Generated by pub on ${DateTime.now()}.'''; // version listing. Use the sha256 from the archive instead. description = description.withSha256(hash); } else { - // TODO(Should we really not fail here???). - if (!bytesEquals(hash, description.sha256)) { - dataError( - 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); - } + assert(bytesEquals(hash, description.sha256), + 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); } } diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index ba37e107c..7413c6f17 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -669,6 +669,11 @@ class HostedSource extends CachedSource { /// /// Validates that the content hash of [id] corresponds to what is already in /// cache, if not the file is redownloaded. + /// + /// If [allowOutdatedHashChecks] is `true` we use a cached version listing + /// response if present instead of probing the server. Not probing allows for + /// `pub get` with a filled cache to be a fast case that doesn't require any + /// new version-listings. @override Future downloadToSystemCache( PackageId id, diff --git a/test/get/enforce_lockfile_test.dart b/test/get/enforce_lockfile_test.dart index 3f9f41fb1..aa9fe2435 100644 --- a/test/get/enforce_lockfile_test.dart +++ b/test/get/enforce_lockfile_test.dart @@ -81,7 +81,7 @@ Future main() async { contains( 'Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( - 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.')), + 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.')), exitCode: DATA, ); }); @@ -102,7 +102,7 @@ Future main() async { await pubGet( args: ['--enforce-lockfile'], error: - 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.', + 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.', exitCode: DATA, ); }); diff --git a/test/testdata/goldens/help_test/pub get --help.txt b/test/testdata/goldens/help_test/pub get --help.txt index 3a34c5d29..28a290960 100644 --- a/test/testdata/goldens/help_test/pub get --help.txt +++ b/test/testdata/goldens/help_test/pub get --help.txt @@ -10,7 +10,7 @@ Usage: pub get [arguments...] network. -n, --dry-run Report what dependencies would change but don't change any. - --enforce-lockfile Only use resolution in existing lockfile. + --enforce-lockfile Only use resolution from existing pubspec.lock. --[no-]precompile Build executables in immediate dependencies. --legacy-packages-file Generate the legacy ".packages" file -C, --directory= Run this in the directory. From d75c4a04c39bcacad94a637bce8b6073e98c657c Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 25 Aug 2022 14:40:04 +0000 Subject: [PATCH 11/51] Remove get enforce-lockfile --- lib/src/command/get.dart | 9 -- lib/src/entrypoint.dart | 16 +-- lib/src/lock_file.dart | 2 +- test/content_hash_test.dart | 2 +- test/get/enforce_lockfile_test.dart | 109 ------------------ .../goldens/help_test/pub get --help.txt | 1 - 6 files changed, 4 insertions(+), 135 deletions(-) delete mode 100644 test/get/enforce_lockfile_test.dart diff --git a/lib/src/command/get.dart b/lib/src/command/get.dart index a73154a4d..be86c4b30 100644 --- a/lib/src/command/get.dart +++ b/lib/src/command/get.dart @@ -7,7 +7,6 @@ import 'dart:async'; import '../command.dart'; import '../log.dart' as log; import '../solver.dart'; -import '../utils.dart'; /// Handles the `get` pub command. class GetCommand extends PubCommand { @@ -29,9 +28,6 @@ class GetCommand extends PubCommand { negatable: false, help: "Report what dependencies would change but don't change any."); - argParser.addFlag('enforce-lockfile', - negatable: false, help: 'Only use resolution from existing pubspec.lock.'); - argParser.addFlag('precompile', help: 'Build executables in immediate dependencies.'); @@ -57,16 +53,12 @@ class GetCommand extends PubCommand { 'The --packages-dir flag is no longer used and does nothing.')); } - if (argResults['dry-run'] && argResults['enforce-lockfile']) { - fail('Cannot do a dry-run with enforce-lockfile.'); - } await entrypoint.acquireDependencies( SolveType.get, dryRun: argResults['dry-run'], precompile: argResults['precompile'], generateDotPackages: argResults['legacy-packages-file'], analytics: analytics, - enforceLockfile: argResults['enforce-lockfile'], ); var example = entrypoint.example; @@ -78,7 +70,6 @@ class GetCommand extends PubCommand { generateDotPackages: argResults['legacy-packages-file'], analytics: analytics, onlyReportSuccessOrFailure: true, - enforceLockfile: argResults['enforce-lockfile'], ); } } diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 7fb20f0f6..77c58be37 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -279,13 +279,7 @@ class Entrypoint { required bool generateDotPackages, required PubAnalytics? analytics, bool onlyReportSuccessOrFailure = false, - bool enforceLockfile = false, }) async { - if (enforceLockfile && !fileExists(lockFilePath)) { - throw ApplicationException( - 'Retrieving dependencies failed. Cannot do `--enforce-lockfile` without an existing `pubspec.lock`.'); - } - if (!onlyReportSuccessOrFailure && hasPubspecOverrides) { log.warning( 'Warning: pubspec.yaml has overrides from $pubspecOverridesPath'); @@ -334,10 +328,6 @@ class Entrypoint { 'by setting it to `quiet`.')); } } - if (enforceLockfile) { - await result.enforceLockfile(); - } - if (!onlyReportSuccessOrFailure) { await result.showReport(type, cache); } @@ -345,12 +335,10 @@ class Entrypoint { await cache.downloadPackages( root, result.packages, - allowOutdatedHashChecks: !enforceLockfile, + allowOutdatedHashChecks: true, ); result.lockFile.checkContentHashes(cache); - if (!enforceLockfile) { - result.lockFile.writeToFile(lockFilePath, cache); - } + result.lockFile.writeToFile(lockFilePath, cache); } if (onlyReportSuccessOrFailure) { log.message('Got dependencies$suffix.'); diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index d314938a1..77b3ac919 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -8,11 +8,11 @@ import 'dart:typed_data'; import 'package:collection/collection.dart' hide mapMap; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; -import 'package:pub/src/command_runner.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:source_span/source_span.dart'; import 'package:yaml/yaml.dart'; +import 'command_runner.dart'; import 'io.dart'; import 'language_version.dart'; import 'log.dart' as log; diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 56c100191..9265db821 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -82,7 +82,7 @@ Future main() async { error: allOf( contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( - 'Cache entry for foo-1.0.0 does not have content-hash matching lockfile.'), + 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.'), ), exitCode: exit_codes.DATA, ); diff --git a/test/get/enforce_lockfile_test.dart b/test/get/enforce_lockfile_test.dart deleted file mode 100644 index aa9fe2435..000000000 --- a/test/get/enforce_lockfile_test.dart +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:io'; - -import 'package:path/path.dart' as p; -import 'package:pub/src/exit_codes.dart'; -import 'package:test/test.dart'; - -import '../descriptor.dart'; -import '../test_pub.dart'; - -Future main() async { - test('Recreates .dart_tool/package_config.json, redownloads archives', - () async { - final server = await servePackages(); - server.serve('foo', '1.0.0'); - await appDir({'foo': 'any'}).create(); - await pubGet(); - final packageConfig = - File(path(p.join(appPath, '.dart_tool', 'package_config.json'))); - packageConfig.deleteSync(); - await runPub(args: ['cache', 'clean', '-f']); - await pubGet(args: ['--enforce-lockfile']); - expect(packageConfig.existsSync(), isTrue); - await cacheDir({'foo': '1.0.0'}).validate(); - await appPackageConfigFile([ - packageConfigEntry(name: 'foo', version: '1.0.0'), - ]).validate(); - }); - - test('Refuses to get if no lockfile exists', () async { - await appDir({}).create(); - await pubGet( - args: ['--enforce-lockfile'], - error: - 'Retrieving dependencies failed. Cannot do `--enforce-lockfile` without an existing `pubspec.lock`.'); - }); - - test('Refuses to get if lockfile is missing package', () async { - final server = await servePackages(); - server.serve('foo', '1.0.0'); - await appDir({}).create(); - await pubGet(); - await appDir({'foo': 'any'}).create(); - - await pubGet( - args: ['--enforce-lockfile'], - error: 'Dependency foo is not already locked in `pubspec.lock`.'); - }); - - test('Refuses to get if package is locked to version not matching constraint', - () async { - final server = await servePackages(); - server.serve('foo', '1.0.0'); - server.serve('foo', '2.0.0'); - await appDir({'foo': '^1.0.0'}).create(); - await pubGet(); - await appDir({'foo': '^2.0.0'}).create(); - await pubGet( - args: ['--enforce-lockfile'], - error: - 'Dependency foo is locked to foo 1.0.0 in `pubspec.lock` but resolves to foo 2.0.0.'); - }); - - test("Refuses to get if hash on server doesn't correspond to lockfile", - () async { - final server = await servePackages(); - server.serveContentHashes = true; - server.serve('foo', '1.0.0'); - await appDir({'foo': '^1.0.0'}).create(); - await pubGet(); - server.serve('foo', '1.0.0', contents: [ - file('README.md', 'Including this will change the content-hash.'), - ]); - - await pubGet( - args: ['--enforce-lockfile'], - error: allOf( - contains( - 'Cached version of foo-1.0.0 has wrong hash - redownloading.'), - contains( - 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.')), - exitCode: DATA, - ); - }); - - test( - 'Refuses to get if archive on legacy server doesn\'t have hash corresponding to lockfile', - () async { - final server = await servePackages(); - server.serveContentHashes = false; - server.serve('foo', '1.0.0'); - await appDir({'foo': '^1.0.0'}).create(); - await pubGet(); - await runPub(args: ['cache', 'clean', '-f']); - server.serve('foo', '1.0.0', contents: [ - file('README.md', 'Including this will change the content-hash.'), - ]); - - await pubGet( - args: ['--enforce-lockfile'], - error: - 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.', - exitCode: DATA, - ); - }); -} diff --git a/test/testdata/goldens/help_test/pub get --help.txt b/test/testdata/goldens/help_test/pub get --help.txt index 28a290960..104f69cd5 100644 --- a/test/testdata/goldens/help_test/pub get --help.txt +++ b/test/testdata/goldens/help_test/pub get --help.txt @@ -10,7 +10,6 @@ Usage: pub get [arguments...] network. -n, --dry-run Report what dependencies would change but don't change any. - --enforce-lockfile Only use resolution from existing pubspec.lock. --[no-]precompile Build executables in immediate dependencies. --legacy-packages-file Generate the legacy ".packages" file -C, --directory= Run this in the directory. From 09aa64bab05a54e52217c88f607ffbf7c3986272 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 23 Aug 2022 12:08:39 +0000 Subject: [PATCH 12/51] Use summary to explain differences if --enforce-lockfile fails --- lib/src/entrypoint.dart | 1 + lib/src/lock_file.dart | 6 +++--- lib/src/solver/result.dart | 33 ++++++++++++++++++++++++--------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 77c58be37..990c5e52a 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -328,6 +328,7 @@ class Entrypoint { 'by setting it to `quiet`.')); } } + if (!onlyReportSuccessOrFailure) { await result.showReport(type, cache); } diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 77b3ac919..868262d2b 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -330,8 +330,8 @@ Generated by pub on ${DateTime.now()}.'''; 'No hash of downloaded archive for ${id.name}-${id.version} found'); } else if (description.sha256 == null) { // We have resolved from a server without archive_sha256 in the - // version listing. Use the sha256 from the archive instead. - description = description.withSha256(cachedHash); + // version listing. + } else { // TODO(Should we really not fail here???). if (!bytesEquals(cachedHash, description.sha256)) { @@ -340,7 +340,7 @@ Cache entry for ${id.name}-${id.version} does not have content-hash matching pub This might indicate -* A corruption of your cache (try running `$topLevelProgram pub cache clean`). +* A corruption of your cache (try running `$topLevelProgram pub cache repair`). * A corruption of the package repository (the package archive has been changed on the server). * The pubspec.lock has been corrupted. diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index abfb8ba45..326ed21e8 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -96,27 +96,42 @@ class SolveResult { /// Checks that the SolveResult is compatible with [_previousLockfile] /// - /// Throws if pubspec.yaml isn't satisfied. - Future enforceLockfile() async { + /// Throws [ApplicationException] if pubspec.yaml isn't satisfied. + Future enforceLockfile(SystemCache cache, + {required bool dryRun}) async { + Never resolutionChanged(String detail) { + SolveReport(SolveType.get, _root, _previousLockFile, this, cache) + .summarize(dryRun: dryRun); + fail('Could not resolve to the exact same resolution. $detail'); + } + + for (final oldPackage in _previousLockFile.packages.keys) { + if (!packages.any((id) => id.name == oldPackage)) { + resolutionChanged('$oldPackage is no longer needed.'); + } + } for (final package in packages) { if (package.isRoot) continue; final previousPackage = _previousLockFile.packages[package.name]; if (previousPackage == null) { - throw ApplicationException( - 'Dependency ${package.name} is not already locked in `pubspec.lock`.'); + resolutionChanged( + 'The dependency ${package.name} is not already locked in `pubspec.lock`.'); } if (previousPackage != package) { - throw ApplicationException( - 'Dependency ${package.name} is locked to $previousPackage in `pubspec.lock` but resolves to $package.'); + resolutionChanged( + 'Dependency ${package.name} is locked to $previousPackage in `pubspec.lock` but now resolves to $package.'); } final previousDescription = previousPackage.description; final newDescription = package.description; if (previousDescription is ResolvedHostedDescription) { final newHash = (newDescription as ResolvedHostedDescription).sha256; - - if (!bytesEquals(previousDescription.sha256, newHash)) { - throw ApplicationException( + // If there was no content-hash in the lock-file we just continue + // silently. + final hadContentHash = previousDescription.sha256 != null; + if (hadContentHash && + !bytesEquals(previousDescription.sha256, newHash)) { + fail( 'Dependency ${package.name} is locked to a different content-hash than what was resolved.'); } } From 7ece91e086cc52252e3a0d3668310ea4c793e347 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 23 Aug 2022 13:00:53 +0000 Subject: [PATCH 13/51] Refactor checkContentHashes logic --- lib/src/global_packages.dart | 10 ++------ lib/src/lock_file.dart | 47 ++++++++++++++---------------------- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index 9b272fbe9..ea2143275 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -179,7 +179,7 @@ class GlobalPackages { final tempDir = cache.createTempDir(); // TODO(rnystrom): Look in "bin" and display list of binaries that // user can run. - _writeLockFile(tempDir, LockFile([id]), cache); + LockFile([id]).writeToFile(p.join(tempDir, 'pubspec.lock'), cache); tryDeleteEntry(_packageDir(name)); tryRenameDir(tempDir, _packageDir(name)); @@ -242,7 +242,7 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam final lockFile = result.lockFile; final tempDir = cache.createTempDir(); - _writeLockFile(tempDir, lockFile, cache); + lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache); // Load the package graph from [result] so we don't need to re-parse all // the pubspecs. @@ -278,12 +278,6 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam if (!silent) log.message('Activated ${_formatPackage(id)}.'); } - /// Finishes activating package [package] by saving [lockFile] in the cache. - void _writeLockFile(String dir, LockFile lockFile, SystemCache cache) { - lockFile.checkContentHashes(cache); - writeTextFile(p.join(dir, 'pubspec.lock'), lockFile.serialize(null, cache)); - } - /// Shows the user the currently active package with [name], if any. LockFile? _describeActive(String name, SystemCache cache) { late final LockFile lockFile; diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 868262d2b..8e1e58510 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -316,8 +316,11 @@ Generated by pub on ${DateTime.now()}.'''; /// Throws a [DataException] if the content-hash of some hosted package locked /// in this lock file differs from the one in the [cache]. - void checkContentHashes(SystemCache cache) { - for (final id in packages.values) { + /// + /// Returns a list of the packages where the contentHash from the cache is + /// inserted into the description. + List checkContentHashes(SystemCache cache) { + return packages.values.map((id) { var description = id.description; if (description is ResolvedHostedDescription) { Uint8List? cachedHash = @@ -330,11 +333,13 @@ Generated by pub on ${DateTime.now()}.'''; 'No hash of downloaded archive for ${id.name}-${id.version} found'); } else if (description.sha256 == null) { // We have resolved from a server without archive_sha256 in the - // version listing. - + // version listing. Use the sha256 from the archive instead. + return PackageId( + id.name, id.version, description.withSha256(cachedHash)); } else { - // TODO(Should we really not fail here???). if (!bytesEquals(cachedHash, description.sha256)) { + // TODO(sigurdm): The error message would read better if the hash + // was compared to the hash from the package-listing. dataError(''' Cache entry for ${id.name}-${id.version} does not have content-hash matching pubspec.lock. @@ -343,12 +348,15 @@ This might indicate * A corruption of your cache (try running `$topLevelProgram pub cache repair`). * A corruption of the package repository (the package archive has been changed on the server). -* The pubspec.lock has been corrupted. +* The pubspec.lock has an invalid content-hash. + +See: $contentHashDocumentationUrl. '''); } } } - } + return id; + }).toList(); } /// Returns the serialized YAML text of the lock file. @@ -357,34 +365,15 @@ This might indicate /// serialize relative path package descriptions. If it is null, they will be /// serialized as absolute. String serialize(String? packageDir, SystemCache cache) { + final packages = checkContentHashes(cache); // Convert the dependencies to a simple object. var packageMap = {}; - for (final id in packages.values) { - var description = id.description; - if (description is ResolvedHostedDescription) { - Uint8List? hash = - description.description.source.sha256FromCache(id, cache); - if (hash == null) { - // This can happen if we resolve from a server not providing hashes, - // but we are not downloading the archives. (eg. for a - // dependency_services run). - // TODO(sigurdm): What should we do here? - log.fine('No hash of downloaded archive found'); - } else if (description.sha256 == null) { - // We have resolved from a server without archive_sha256 in the - // version listing. Use the sha256 from the archive instead. - description = description.withSha256(hash); - } else { - assert(bytesEquals(hash, description.sha256), - 'Cache entry for ${id.name}-${id.version} does not have content-hash matching lockfile.'); - } - } - + for (final id in packages) { packageMap[id.name] = { 'version': id.version.toString(), 'source': id.source.name, 'description': - description.serializeForLockfile(containingDir: packageDir), + id.description.serializeForLockfile(containingDir: packageDir), 'dependency': _dependencyType(id.name) }; } From 763dca97ab86f9cde707fd4ab664b260a0cc20fe Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 25 Aug 2022 14:56:26 +0000 Subject: [PATCH 14/51] Add missing member --- lib/src/lock_file.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index 8e1e58510..b8046ff92 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -45,6 +45,9 @@ class LockFile { /// `dependency_overrides` section. final Set overriddenDependencies; + static const contentHashesDocumentationUrl = + 'https://dart.dev/go/content-hashes'; + /// Creates a new lockfile containing [ids]. /// /// If passed, [mainDependencies], [devDependencies], and @@ -350,7 +353,7 @@ This might indicate on the server). * The pubspec.lock has an invalid content-hash. -See: $contentHashDocumentationUrl. +See: $contentHashesDocumentationUrl. '''); } } From 1144d02d9f71537116038df3d0fbb3d46741d50c Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 30 Aug 2022 12:13:09 +0000 Subject: [PATCH 15/51] WIP --- lib/src/entrypoint.dart | 30 ++++-- lib/src/global_packages.dart | 28 +++-- lib/src/lock_file.dart | 55 +--------- lib/src/solver/report.dart | 42 ++++---- lib/src/solver/result.dart | 101 ++++++++---------- lib/src/source/hosted.dart | 24 ++++- test/content_hash_test.dart | 42 +++++++- ...--verbose and on unexpected exceptions.txt | 10 +- 8 files changed, 167 insertions(+), 165 deletions(-) diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 990c5e52a..e4cceb3ae 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -31,6 +31,7 @@ import 'pub_embeddable_command.dart'; import 'pubspec.dart'; import 'sdk.dart'; import 'solver.dart'; +import 'solver/report.dart'; import 'source/cached.dart'; import 'source/unknown.dart'; import 'system_cache.dart'; @@ -309,7 +310,6 @@ class Entrypoint { rethrow; } } - _lockFile = result.lockFile; // Log once about all overridden packages. if (warnAboutPreReleaseSdkOverrides) { @@ -329,22 +329,32 @@ class Entrypoint { } } + // We have to download files also with --dry-run to ensure we know the + // archive hashes for downloaded files. + final newLockFile = + await result.downloadPackages(cache, allowOutdatedHashChecks: true); + + final report = SolveReport( + type, + root, + lockFile, + newLockFile, + result.availableVersions, + cache, + ); if (!onlyReportSuccessOrFailure) { - await result.showReport(type, cache); + await report.show(); } + _lockFile = newLockFile; + if (!dryRun) { - await cache.downloadPackages( - root, - result.packages, - allowOutdatedHashChecks: true, - ); - result.lockFile.checkContentHashes(cache); - result.lockFile.writeToFile(lockFilePath, cache); + newLockFile.writeToFile(lockFilePath, cache); } + if (onlyReportSuccessOrFailure) { log.message('Got dependencies$suffix.'); } else { - await result.summarizeChanges(type, cache, dryRun: dryRun); + report.summarize(dryRun: dryRun); } if (!dryRun) { diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index ea2143275..0925840ce 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -23,6 +23,7 @@ import 'sdk.dart'; import 'sdk/dart.dart'; import 'solver.dart'; import 'solver/incompatibility_cause.dart'; +import 'solver/report.dart'; import 'source/cached.dart'; import 'source/git.dart'; import 'source/hosted.dart'; @@ -224,10 +225,14 @@ class GlobalPackages { // We want the entrypoint to be rooted at 'dep' not the dummy-package. result.packages.removeWhere((id) => id.name == 'pub global activate'); - final sameVersions = originalLockFile != null && - originalLockFile.samePackageIds(result.lockFile); + final lockFile = await result.downloadPackages( + cache, + allowOutdatedHashChecks: true, + ); + final sameVersions = + originalLockFile != null && originalLockFile.samePackageIds(lockFile); - final PackageId id = result.lockFile.packages[name]!; + final PackageId id = lockFile.packages[name]!; if (sameVersions) { log.message(''' The package $name is already activated at newest available version. @@ -235,12 +240,17 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam '''); } else { // Only precompile binaries if we have a new resolution. - if (!silent) await result.showReport(SolveType.get, cache); - - await cache.downloadPackages(root, result.packages, - allowOutdatedHashChecks: true); + if (!silent) { + await SolveReport( + SolveType.get, + root, + originalLockFile ?? LockFile.empty(), + lockFile, + result.availableVersions, + cache, + ).show(); + } - final lockFile = result.lockFile; final tempDir = cache.createTempDir(); lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache); @@ -265,7 +275,7 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam final entrypoint = Entrypoint.global( _packageDir(id.name), cache.loadCached(id), - result.lockFile, + lockFile, cache, solveResult: result, ); diff --git a/lib/src/lock_file.dart b/lib/src/lock_file.dart index b8046ff92..0eaed1d37 100644 --- a/lib/src/lock_file.dart +++ b/lib/src/lock_file.dart @@ -3,7 +3,6 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:convert'; -import 'dart:typed_data'; import 'package:collection/collection.dart' hide mapMap; import 'package:meta/meta.dart'; @@ -12,15 +11,12 @@ import 'package:pub_semver/pub_semver.dart'; import 'package:source_span/source_span.dart'; import 'package:yaml/yaml.dart'; -import 'command_runner.dart'; import 'io.dart'; import 'language_version.dart'; -import 'log.dart' as log; import 'package_config.dart'; import 'package_name.dart'; import 'packages_file.dart' as packages_file; import 'sdk.dart' show sdk; -import 'source/hosted.dart'; import 'system_cache.dart'; import 'utils.dart'; @@ -45,9 +41,6 @@ class LockFile { /// `dependency_overrides` section. final Set overriddenDependencies; - static const contentHashesDocumentationUrl = - 'https://dart.dev/go/content-hashes'; - /// Creates a new lockfile containing [ids]. /// /// If passed, [mainDependencies], [devDependencies], and @@ -317,61 +310,15 @@ Generated by pub on ${DateTime.now()}.'''; return '${JsonEncoder.withIndent(' ').convert(packageConfig.toJson())}\n'; } - /// Throws a [DataException] if the content-hash of some hosted package locked - /// in this lock file differs from the one in the [cache]. - /// - /// Returns a list of the packages where the contentHash from the cache is - /// inserted into the description. - List checkContentHashes(SystemCache cache) { - return packages.values.map((id) { - var description = id.description; - if (description is ResolvedHostedDescription) { - Uint8List? cachedHash = - description.description.source.sha256FromCache(id, cache); - if (cachedHash == null) { - // This can happen if we resolve from a server not providing hashes, - // but we are not downloading the archives. (eg. for a - // dependency_services run). - log.fine( - 'No hash of downloaded archive for ${id.name}-${id.version} found'); - } else if (description.sha256 == null) { - // We have resolved from a server without archive_sha256 in the - // version listing. Use the sha256 from the archive instead. - return PackageId( - id.name, id.version, description.withSha256(cachedHash)); - } else { - if (!bytesEquals(cachedHash, description.sha256)) { - // TODO(sigurdm): The error message would read better if the hash - // was compared to the hash from the package-listing. - dataError(''' -Cache entry for ${id.name}-${id.version} does not have content-hash matching pubspec.lock. - -This might indicate - -* A corruption of your cache (try running `$topLevelProgram pub cache repair`). -* A corruption of the package repository (the package archive has been changed - on the server). -* The pubspec.lock has an invalid content-hash. - -See: $contentHashesDocumentationUrl. -'''); - } - } - } - return id; - }).toList(); - } - /// Returns the serialized YAML text of the lock file. /// /// [packageDir] is the containing directory of the root package, used to /// serialize relative path package descriptions. If it is null, they will be /// serialized as absolute. String serialize(String? packageDir, SystemCache cache) { - final packages = checkContentHashes(cache); // Convert the dependencies to a simple object. var packageMap = {}; - for (final id in packages) { + for (final id in packages.values) { packageMap[id.name] = { 'version': id.version.toString(), 'source': id.source.name, diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 82555f454..0f826a39f 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -25,21 +25,27 @@ class SolveReport { final SolveType _type; final Package _root; final LockFile _previousLockFile; - final SolveResult _result; + final LockFile _newLockFile; final SystemCache _cache; - /// The dependencies in [_result], keyed by package name. - final _dependencies = {}; + /// The available versions of all selected packages from their source. + /// + /// An entry here may not include the full list of versions available if the + /// given package was locked and did not need to be unlocked during the solve. + /// + /// Version list will not contain any retracted package versions. + final Map> _availableVersions; final _output = StringBuffer(); - SolveReport(this._type, this._root, this._previousLockFile, this._result, - this._cache) { - // Fill the map so we can use it later. - for (var id in _result.packages) { - _dependencies[id.name] = id; - } - } + SolveReport( + this._type, + this._root, + this._previousLockFile, + this._newLockFile, + this._availableVersions, + this._cache, + ); /// Displays a report of the results of the version resolution relative to /// the previous lock file. @@ -54,13 +60,13 @@ class SolveReport { /// If [dryRun] is true, describes it in terms of what would be done. void summarize({bool dryRun = false}) { // Count how many dependencies actually changed. - var dependencies = _dependencies.keys.toSet(); + var dependencies = _newLockFile.packages.keys.toSet(); dependencies.addAll(_previousLockFile.packages.keys); dependencies.remove(_root.name); var numChanged = dependencies.where((name) { var oldId = _previousLockFile.packages[name]; - var newId = _dependencies[name]; + var newId = _newLockFile.packages[name]; // Added or removed dependencies count. if (oldId == null) return true; @@ -107,7 +113,7 @@ class SolveReport { _output.clear(); // Show the new set of dependencies ordered by name. - var names = _result.packages.map((id) => id.name).toList(); + var names = _newLockFile.packages.keys.toList(); names.remove(_root.name); names.sort(); for (final name in names) { @@ -146,7 +152,7 @@ class SolveReport { /// if discontinued packages are detected. Future reportDiscontinued() async { var numDiscontinued = 0; - for (var id in _result.packages) { + for (var id in _newLockFile.packages.values) { if (id.description is RootDescription) continue; final status = await id.source .status(id.toRef(), id.version, _cache, maxAge: Duration(days: 3)); @@ -168,8 +174,8 @@ class SolveReport { /// Displays a two-line message, number of outdated packages and an /// instruction to run `pub outdated` if outdated packages are detected. void reportOutdated() { - final outdatedPackagesCount = _result.packages.where((id) { - final versions = _result.availableVersions[id.name]!; + final outdatedPackagesCount = _newLockFile.packages.values.where((id) { + final versions = _availableVersions[id.name]!; // A version is counted: // - if there is a newer version which is not a pre-release and current // version is also not a pre-release or, @@ -198,7 +204,7 @@ class SolveReport { /// "(override)" next to overridden packages. Future _reportPackage(String name, {bool alwaysShow = false, bool highlightOverride = true}) async { - var newId = _dependencies[name]; + var newId = _previousLockFile.packages[name]; var oldId = _previousLockFile.packages[name]; var id = newId ?? oldId!; @@ -245,7 +251,7 @@ class SolveReport { // See if there are any newer versions of the package that we were // unable to upgrade to. if (newId != null && _type != SolveType.downgrade) { - var versions = _result.availableVersions[newId.name]!; + var versions = _availableVersions[newId.name]!; var newerStable = false; var newerUnstable = false; diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 326ed21e8..90e82cafb 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:typed_data'; + import 'package:collection/collection.dart'; import 'package:pub_semver/pub_semver.dart'; @@ -50,9 +52,19 @@ class SolveResult { /// The wall clock time the resolution took. final Duration resolutionTime; - /// The [LockFile] representing the packages selected by this version - /// resolution. - LockFile get lockFile { + /// Downloads all the packages selected by this version. + /// + /// Returns the [LockFile] representing the packages selected by this version + /// resolution. Any resolved ids will correspond + Future downloadPackages( + SystemCache cache, { + required bool allowOutdatedHashChecks, + }) async { + await cache.downloadPackages( + _root, + packages, + allowOutdatedHashChecks: allowOutdatedHashChecks, + ); // Don't factor in overridden dependencies' SDK constraints, because we'll // accept those packages even if their constraints don't match. var nonOverrides = pubspecs.values @@ -67,12 +79,29 @@ class SolveResult { .intersect(sdkConstraints[identifier] ?? VersionConstraint.any); }); } - - return LockFile(packages, - sdkConstraints: sdkConstraints, - mainDependencies: MapKeySet(_root.dependencies), - devDependencies: MapKeySet(_root.devDependencies), - overriddenDependencies: MapKeySet(_root.dependencyOverrides)); + return LockFile( + packages.map((id) { + var description = id.description; + // Use the cached content-hashes after downloading to ensure that + // content-hashes from legacy servers gets used. + if (description is ResolvedHostedDescription) { + Uint8List? cachedHash = + description.description.source.sha256FromCache(id, cache); + if (cachedHash == null) { + // This should not happen. + throw StateError( + 'Archive for ${id.name}-${id.version} has no content hash.'); + } + return PackageId( + id.name, id.version, description.withSha256(cachedHash)); + } + return id; + }).toList(), + sdkConstraints: sdkConstraints, + mainDependencies: MapKeySet(_root.dependencies), + devDependencies: MapKeySet(_root.devDependencies), + overriddenDependencies: MapKeySet(_root.dependencyOverrides), + ); } final LockFile _previousLockFile; @@ -94,55 +123,12 @@ class SolveResult { SolveResult(this._root, this._previousLockFile, this.packages, this.pubspecs, this.availableVersions, this.attemptedSolutions, this.resolutionTime); - /// Checks that the SolveResult is compatible with [_previousLockfile] - /// - /// Throws [ApplicationException] if pubspec.yaml isn't satisfied. - Future enforceLockfile(SystemCache cache, - {required bool dryRun}) async { - Never resolutionChanged(String detail) { - SolveReport(SolveType.get, _root, _previousLockFile, this, cache) - .summarize(dryRun: dryRun); - fail('Could not resolve to the exact same resolution. $detail'); - } - - for (final oldPackage in _previousLockFile.packages.keys) { - if (!packages.any((id) => id.name == oldPackage)) { - resolutionChanged('$oldPackage is no longer needed.'); - } + /// Warns if the content-hash of some hosted package locked + /// in this [lockFile] differs from the one in the [cache]. + Future checkContentHashes(LockFile lockFile, SystemCache cache) { + for (final package in lockFile.packages.values) { + if (package.description is ResolvedHostedDescription) {} } - for (final package in packages) { - if (package.isRoot) continue; - final previousPackage = _previousLockFile.packages[package.name]; - if (previousPackage == null) { - resolutionChanged( - 'The dependency ${package.name} is not already locked in `pubspec.lock`.'); - } - if (previousPackage != package) { - resolutionChanged( - 'Dependency ${package.name} is locked to $previousPackage in `pubspec.lock` but now resolves to $package.'); - } - final previousDescription = previousPackage.description; - final newDescription = package.description; - - if (previousDescription is ResolvedHostedDescription) { - final newHash = (newDescription as ResolvedHostedDescription).sha256; - // If there was no content-hash in the lock-file we just continue - // silently. - final hadContentHash = previousDescription.sha256 != null; - if (hadContentHash && - !bytesEquals(previousDescription.sha256, newHash)) { - fail( - 'Dependency ${package.name} is locked to a different content-hash than what was resolved.'); - } - } - } - } - - /// Displays a report of what changes were made to the lockfile. - /// - /// [type] is the type of version resolution that was run. - Future showReport(SolveType type, SystemCache cache) async { - await SolveReport(type, _root, _previousLockFile, this, cache).show(); } /// Displays a one-line message summarizing what changes were made (or would @@ -154,7 +140,6 @@ class SolveResult { /// [type] is the type of version resolution that was run. Future summarizeChanges(SolveType type, SystemCache cache, {bool dryRun = false}) async { - final report = SolveReport(type, _root, _previousLockFile, this, cache); report.summarize(dryRun: dryRun); if (type == SolveType.upgrade) { await report.reportDiscontinued(); diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 7413c6f17..113ef8997 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -31,6 +31,8 @@ import '../system_cache.dart'; import '../utils.dart'; import 'cached.dart'; +const contentHashesDocumentationUrl = 'https://dart.dev/go/content-hashes'; + /// Validates and normalizes a [hostedUrl] which is pointing to a pub server. /// /// A [hostedUrl] is a URL pointing to a _hosted pub server_ as defined by the @@ -959,7 +961,7 @@ class HostedSource extends CachedSource { var url = versionInfo.archiveUrl; log.io('Get package from $url.'); - log.message('Downloading ${log.bold(id.name)} ${id.version}...'); + log.fine('Downloading ${log.bold(id.name)} ${id.version}...'); // Download and extract the archive to a temp directory. await withTempDir((tempDirForArchive) async { @@ -984,9 +986,14 @@ class HostedSource extends CachedSource { final actualHash = output.value; if (expectedHash != null && output.value != expectedHash) { log.fine( - 'Expected content-hash $expectedHash actual: ${output.value}.'); - throw FormatException( - 'Downloaded archive for ${id.name}-${id.version} had wrong content-hash.'); + 'Expected content-hash for ${id.name}-${id.version} $expectedHash actual: ${output.value}.'); + throw FormatException(''' +Downloaded archive for ${id.name}-${id.version} had wrong content-hash. + +This indicates a problem on the package repository: `${description.url}`. + +See $contentHashesDocumentationUrl. +'''); } final path = hashPath(id, cache); ensureDir(p.dirname(path)); @@ -1002,10 +1009,17 @@ class HostedSource extends CachedSource { // cancelling a http stream makes it not reusable. // There are ways around this, and we might revisit this later. Stream> stream = response.stream; + + // It is important that we do not compare against id.description.sha256, + // as we need to check against the newly fetched version listing to ensure + // that content changes result in updated lockfiles, not failure to + // download. final expectedSha256 = versionInfo.archiveSha256; stream = checkSha256( - stream, (expectedSha256 == null) ? null : Digest(expectedSha256)); + stream, + (expectedSha256 == null) ? null : Digest(expectedSha256), + ); await createFileFromStream(stream, archivePath); final tempDir = cache.createTempDir(); diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 9265db821..8a718bf23 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -64,7 +64,8 @@ Future main() async { ); }); - test('If content is updated on server we refuse to continue', () async { + test('If content is updated on server we warn and update the lockfile', + () async { final server = await servePackages(); server.serveContentHashes = true; server.serve('foo', '1.0.0'); @@ -74,18 +75,49 @@ Future main() async { contents: [file('new_file.txt', 'This file could be malicious.')]); // Pub get will not revisit the file-listing if everything resolves, and only compare with a cached value. await pubGet(); - // Deleting the version-listing cache will cause it to be refetched, and the error will happen. + // Deleting the version-listing cache will cause it to be refetched, and the + // warning will happen. File(p.join(globalServer.cachingPath, '.cache', 'foo-versions.json')) .deleteSync(); - await pubGet( - error: allOf( + warning: allOf( contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.'), ), - exitCode: exit_codes.DATA, + exitCode: exit_codes.SUCCESS, ); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final newHash = lockfile['packages']['foo']['description']['sha256']; + expect(newHash, await server.getSha256('foo', '1.0.0')); + }); + + test( + 'If content is updated on legacy server, and the download needs refreshing we warn and update the lockfile', + () async { + final server = await servePackages(); + server.serveContentHashes = false; + server.serve('foo', '1.0.0'); + await appDir({'foo': 'any'}).create(); + await pubGet(); + server.serve('foo', '1.0.0', + contents: [file('new_file.txt', 'This file could be malicious.')]); + // Deleting the hash-file cache will cause it to be refetched, and the + // warning will happen. + File(p.join(globalServer.cachingPath, '.hashes', 'foo-1.0.0.sha256')) + .deleteSync(); + + await pubGet( + warning: contains( + 'Content of foo-1.0.0 has changed compared to your previous pubspec.lock.', + ), + exitCode: exit_codes.SUCCESS, + ); + final lockfile = loadYaml( + File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); + final newHash = lockfile['packages']['foo']['description']['sha256']; + expect(newHash, server.getSha256('foo', '1.0.0')); }); test( diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index bab4187f1..4f3cf82ee 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -23,13 +23,13 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | user-agent: Dart pub 0.1.2+3 [E] IO : HTTP response 200 OK for GET http://localhost:$PORT/api/packages/foo [E] | took: $TIME +[E] | x-powered-by: Dart with package:shelf [E] | date: $TIME [E] | content-length: 197 [E] | x-frame-options: SAMEORIGIN [E] | content-type: text/plain; charset=utf-8 [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff -[E] | server: dart:io with Shelf [E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. [E] FINE: Contents: [E] | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz"}],"_fetchedAt": "$TIME"} @@ -48,13 +48,13 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | user-agent: Dart pub 0.1.2+3 [E] IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz [E] | took: $TIME +[E] | x-powered-by: Dart with package:shelf [E] | transfer-encoding: chunked [E] | date: $TIME [E] | x-frame-options: SAMEORIGIN [E] | content-type: text/plain; charset=utf-8 [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff -[E] | server: dart:io with Shelf [E] IO : Creating $FILE from stream [E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. [E] FINE: Contents: @@ -164,13 +164,13 @@ IO : HTTP GET http://localhost:$PORT/api/packages/foo | user-agent: Dart pub 0.1.2+3 IO : HTTP response 200 OK for GET http://localhost:$PORT/api/packages/foo | took: $TIME + | x-powered-by: Dart with package:shelf | date: $TIME | content-length: 197 | x-frame-options: SAMEORIGIN | content-type: text/plain; charset=utf-8 | x-xss-protection: 1; mode=block | x-content-type-options: nosniff - | server: dart:io with Shelf IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. FINE: Contents: | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz"}],"_fetchedAt": "$TIME"} @@ -191,13 +191,13 @@ IO : HTTP GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz | user-agent: Dart pub 0.1.2+3 IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz | took: $TIME + | x-powered-by: Dart with package:shelf | transfer-encoding: chunked | date: $TIME | x-frame-options: SAMEORIGIN | content-type: text/plain; charset=utf-8 | x-xss-protection: 1; mode=block | x-content-type-options: nosniff - | server: dart:io with Shelf IO : Creating $FILE from stream IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. FINE: Contents: @@ -258,7 +258,6 @@ $ tool/test-bin/pub_command_runner.dart pub fail [E] Bad state: Pub has crashed [E] tool/test-bin/pub_command_runner.dart $LINE:$COL ThrowingCommand.runProtected [E] package:pub/src/command.dart $LINE:$COL PubCommand.run. -[E] package:pub/src/command.dart $LINE:$COL PubCommand.run. [E] dart:async new Future.sync [E] package:pub/src/utils.dart $LINE:$COL captureErrors.wrappedCallback [E] dart:async runZonedGuarded @@ -297,7 +296,6 @@ FINE: Pub 0.1.2+3 ERR : Bad state: Pub has crashed FINE: Exception type: StateError ERR : tool/test-bin/pub_command_runner.dart $LINE:$COL ThrowingCommand.runProtected - | package:pub/src/command.dart $LINE:$COL PubCommand.run. | package:pub/src/command.dart $LINE:$COL PubCommand.run. | dart:async new Future.sync | package:pub/src/utils.dart $LINE:$COL captureErrors.wrappedCallback From e6ea414141a0028ceaa61757cd86a545713d9d3e Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 1 Sep 2022 11:19:59 +0000 Subject: [PATCH 16/51] Do hash-checks in result --- lib/src/command/dependency_services.dart | 16 ++-- lib/src/entrypoint.dart | 6 +- lib/src/global_packages.dart | 2 +- lib/src/solver/report.dart | 26 ++++-- lib/src/solver/result.dart | 83 +++++++++++-------- lib/src/system_cache.dart | 20 ----- .../adds_latest_matching_version_test.dart | 2 +- test/cache/add/adds_latest_version_test.dart | 4 +- .../all_adds_all_matching_versions_test.dart | 12 +-- .../all_with_some_versions_present_test.dart | 18 ++-- test/cache/add/already_cached_test.dart | 3 +- test/cache/repair/handles_failure_test.dart | 3 - test/cache/repair/hosted.dart | 7 +- .../repair/recompiles_snapshots_test.dart | 1 - test/cache/repair/updates_binstubs_test.dart | 1 - test/content_hash_test.dart | 6 +- .../activate_git_after_hosted_test.dart | 2 +- .../activate_hosted_after_git_test.dart | 3 +- .../activate_hosted_after_path_test.dart | 4 +- .../activate/activate_hosted_twice_test.dart | 4 +- .../activate/custom_hosted_url_test.dart | 15 ++-- .../activate/different_version_test.dart | 3 +- .../activate/ignores_active_version_test.dart | 3 +- .../installs_dependencies_for_git_test.dart | 2 +- .../installs_dependencies_for_path_test.dart | 2 - .../activate/installs_dependencies_test.dart | 2 +- .../reactivating_git_upgrades_test.dart | 2 +- .../activate/uncached_package_test.dart | 1 - ...eactivate_and_reactivate_package_test.dart | 6 +- test/test_pub.dart | 5 +- .../embedding_test/--color forces colors.txt | 1 - ...--verbose and on unexpected exceptions.txt | 6 +- .../dry_run_does_not_apply_changes_test.dart | 2 +- tool/test.dart | 2 +- 34 files changed, 144 insertions(+), 131 deletions(-) diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index 527358425..d49c3dbce 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -477,18 +477,24 @@ class DependencyServicesApplyCommand extends PubCommand { // `pub get` gets this version-listing from the downloaded archive // but we don't want to download all archives - so we copy it from // the version listing. - final listedId = (await cache.getVersions(package.toRef())) + var listedId = (await cache.getVersions(package.toRef())) .firstWhere((id) => id == package); if ((listedId.description as ResolvedHostedDescription).sha256 == null) { - // This happens when we resolved a package from a server not - // providing archive_sha256. As a side-effect of downloading - // the package we compute and store the sha256, and that will - // be picked up by entrypoint.saveLockFile. + // This happens when we resolved a package from a legacy server + // not providing archive_sha256. As a side-effect of downloading + // the package we compute and store the sha256. await cache.downloadPackage( package, allowOutdatedHashChecks: false, ); + listedId = PackageId( + listedId.name, + listedId.version, + description.withSha256( + cache.hosted.sha256FromCache(listedId, cache), + ), + ); } updatedPackages.add(listedId); } else { diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 0d61093c3..a08c245c7 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -356,8 +356,8 @@ class Entrypoint { // We have to download files also with --dry-run to ensure we know the // archive hashes for downloaded files. - final newLockFile = - await result.downloadPackages(cache, allowOutdatedHashChecks: true); + final newLockFile = await result.downloadCachedPackages(cache, + allowOutdatedHashChecks: true); final report = SolveReport( type, @@ -379,7 +379,7 @@ class Entrypoint { if (onlyReportSuccessOrFailure) { log.message('Got dependencies$suffix.'); } else { - report.summarize(dryRun: dryRun); + await report.summarize(dryRun: dryRun); } if (!dryRun) { diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index 0cb60e408..fbec3c17d 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -224,7 +224,7 @@ class GlobalPackages { // We want the entrypoint to be rooted at 'dep' not the dummy-package. result.packages.removeWhere((id) => id.name == 'pub global activate'); - final lockFile = await result.downloadPackages( + final lockFile = await result.downloadCachedPackages( cache, allowOutdatedHashChecks: true, ); diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 0f826a39f..4307bd0b8 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -47,8 +47,8 @@ class SolveReport { this._cache, ); - /// Displays a report of the results of the version resolution relative to - /// the previous lock file. + /// Displays a report of the results of the version resolution in + /// [_newLockFile] relative to the [_previousLockFile] file. Future show() async { await _reportChanges(); await _reportOverrides(); @@ -58,7 +58,13 @@ class SolveReport { /// be made) to the lockfile. /// /// If [dryRun] is true, describes it in terms of what would be done. - void summarize({bool dryRun = false}) { + /// + /// [type] is the type of version resolution that was run. + + /// If [type] is `SolveType.UPGRADE` it also shows the number of packages that + /// are not at the latest available version and the number of outdated + /// packages. + Future summarize({bool dryRun = false}) async { // Count how many dependencies actually changed. var dependencies = _newLockFile.packages.keys.toSet(); dependencies.addAll(_previousLockFile.packages.keys); @@ -105,6 +111,10 @@ class SolveReport { log.message('Changed $numChanged dependencies$suffix!'); } } + if (_type == SolveType.upgrade) { + await reportDiscontinued(); + reportOutdated(); + } } /// Displays a report of all of the previous and current dependencies and @@ -204,7 +214,7 @@ class SolveReport { /// "(override)" next to overridden packages. Future _reportPackage(String name, {bool alwaysShow = false, bool highlightOverride = true}) async { - var newId = _previousLockFile.packages[name]; + var newId = _newLockFile.packages[name]; var oldId = _previousLockFile.packages[name]; var id = newId ?? oldId!; @@ -224,6 +234,7 @@ class SolveReport { // + The package was added. // > The package was upgraded from a lower version. // < The package was downgraded from a higher version. + // ~ Package contents has changed, but not the version number. // * Any other change between the old and new package. String icon; if (isOverridden) { @@ -234,7 +245,8 @@ class SolveReport { } else if (oldId == null) { icon = log.green('+ '); addedOrRemoved = true; - } else if (oldId.description != newId.description) { + } else if (oldId.description.description != newId.description.description) { + // Eg. a changed source in pubspec.yaml. icon = log.cyan('* '); changed = true; } else if (oldId.version < newId.version) { @@ -243,6 +255,10 @@ class SolveReport { } else if (oldId.version > newId.version) { icon = log.cyan('< '); changed = true; + } else if (oldId.description != newId.description) { + // Eg. a changed hash or revision. + icon = log.cyan('~ '); + changed = true; } else { // Unchanged. icon = ' '; diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 90e82cafb..88812217c 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -7,7 +7,7 @@ import 'dart:typed_data'; import 'package:collection/collection.dart'; import 'package:pub_semver/pub_semver.dart'; -import '../exceptions.dart'; +import '../http.dart'; import '../io.dart'; import '../lock_file.dart'; import '../log.dart' as log; @@ -15,11 +15,10 @@ import '../package.dart'; import '../package_name.dart'; import '../pub_embeddable_command.dart'; import '../pubspec.dart'; +import '../source/cached.dart'; import '../source/hosted.dart'; import '../system_cache.dart'; import '../utils.dart'; -import 'report.dart'; -import 'type.dart'; /// The result of a successful version resolution. class SolveResult { @@ -52,19 +51,34 @@ class SolveResult { /// The wall clock time the resolution took. final Duration resolutionTime; - /// Downloads all the packages selected by this version. + /// Downloads all the cached packages selected by this version resolution. + /// + /// If some already cached package differs from what is provided by the server + /// (according to the content-hash) a warning is printed and the package is + /// redownloaded. /// /// Returns the [LockFile] representing the packages selected by this version - /// resolution. Any resolved ids will correspond - Future downloadPackages( + /// resolution. Any resolved [PackageId]s will correspond to those in the + /// cache (and thus to the one provided by the server). + /// + /// If there is a mismatch between the previous content-hash from pubspec.lock + /// and the new one a warning will be printed but the new one will be + /// returned. + Future downloadCachedPackages( SystemCache cache, { required bool allowOutdatedHashChecks, }) async { - await cache.downloadPackages( - _root, - packages, - allowOutdatedHashChecks: allowOutdatedHashChecks, - ); + await Future.wait(packages.map((id) async { + if (id.source is CachedSource) { + await withDependencyType(_root.dependencyType(id.name), () async { + await cache.downloadPackage( + id, + allowOutdatedHashChecks: allowOutdatedHashChecks, + ); + }); + } + })); + // Don't factor in overridden dependencies' SDK constraints, because we'll // accept those packages even if their constraints don't match. var nonOverrides = pubspecs.values @@ -92,6 +106,29 @@ class SolveResult { throw StateError( 'Archive for ${id.name}-${id.version} has no content hash.'); } + final originalEntry = _previousLockFile.packages[id.name]; + if (originalEntry != null && originalEntry.version == id.version) { + final originalDescription = originalEntry.description; + if (originalDescription is ResolvedHostedDescription) { + final Uint8List? originalHash = originalDescription.sha256; + if (originalHash != null) { + if (!bytesEquals(cachedHash, originalHash)) { + log.warning(''' +Content of ${id.name}-${id.version} has changed compared to what was locked your pubspec.lock. + +This might indicate that: +* The content has changed on the server since you created the pubspec.lock. +* The pubspec.lock has been corrupted. + +Your pubspec.lock has been updated according to what is on your server. + +See $contentHashesDocumentationUrl for more information. +'''); + } + } + } + } + return PackageId( id.name, id.version, description.withSha256(cachedHash)); } @@ -123,30 +160,6 @@ class SolveResult { SolveResult(this._root, this._previousLockFile, this.packages, this.pubspecs, this.availableVersions, this.attemptedSolutions, this.resolutionTime); - /// Warns if the content-hash of some hosted package locked - /// in this [lockFile] differs from the one in the [cache]. - Future checkContentHashes(LockFile lockFile, SystemCache cache) { - for (final package in lockFile.packages.values) { - if (package.description is ResolvedHostedDescription) {} - } - } - - /// Displays a one-line message summarizing what changes were made (or would - /// be made) to the lockfile. - /// - /// If [type] is `SolveType.UPGRADE` it also shows the number of packages - /// that are not at the latest available version. - /// - /// [type] is the type of version resolution that was run. - Future summarizeChanges(SolveType type, SystemCache cache, - {bool dryRun = false}) async { - report.summarize(dryRun: dryRun); - if (type == SolveType.upgrade) { - await report.reportDiscontinued(); - report.reportOutdated(); - } - } - /// Send analytics about the package resolution. void sendAnalytics(PubAnalytics pubAnalytics) { ArgumentError.checkNotNull(pubAnalytics); diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart index 19bc8b250..3fe617e1a 100644 --- a/lib/src/system_cache.dart +++ b/lib/src/system_cache.dart @@ -10,7 +10,6 @@ import 'package:pub_semver/pub_semver.dart'; import 'authentication/token_store.dart'; import 'exceptions.dart'; -import 'http.dart'; import 'io.dart'; import 'io.dart' as io show createTempDir; import 'log.dart' as log; @@ -264,25 +263,6 @@ class SystemCache { return latest; } - - /// Downloads all cached packages in [packages]. - Future downloadPackages( - Package root, - List packages, { - required bool allowOutdatedHashChecks, - }) async { - await Future.wait(packages.map((id) async { - if (id.source is! CachedSource) { - return; - } - return await withDependencyType(root.dependencyType(id.name), () async { - await downloadPackage( - id, - allowOutdatedHashChecks: allowOutdatedHashChecks, - ); - }); - })); - } } typedef SourceRegistry = Source Function(String? name); diff --git a/test/cache/add/adds_latest_matching_version_test.dart b/test/cache/add/adds_latest_matching_version_test.dart index 7bf962569..9e06bf792 100644 --- a/test/cache/add/adds_latest_matching_version_test.dart +++ b/test/cache/add/adds_latest_matching_version_test.dart @@ -21,8 +21,8 @@ void main() { await runPub( args: ['cache', 'add', 'foo', '-v', '>=1.0.0 <2.0.0'], - output: 'Downloading foo 1.2.3...', silent: allOf([ + contains('Downloading foo 1.2.3...'), contains('X-Pub-OS: ${Platform.operatingSystem}'), contains('X-Pub-Command: cache add'), contains('X-Pub-Session-ID:'), diff --git a/test/cache/add/adds_latest_version_test.dart b/test/cache/add/adds_latest_version_test.dart index cf34857e2..d3020399c 100644 --- a/test/cache/add/adds_latest_version_test.dart +++ b/test/cache/add/adds_latest_version_test.dart @@ -15,7 +15,9 @@ void main() { ..serve('foo', '1.2.4-dev'); await runPub( - args: ['cache', 'add', 'foo'], output: 'Downloading foo 1.2.3...'); + args: ['cache', 'add', 'foo'], + silent: contains('Downloading foo 1.2.3...'), + ); await d.cacheDir({'foo': '1.2.3'}).validate(); }); diff --git a/test/cache/add/all_adds_all_matching_versions_test.dart b/test/cache/add/all_adds_all_matching_versions_test.dart index a05e8dca5..b7c884f97 100644 --- a/test/cache/add/all_adds_all_matching_versions_test.dart +++ b/test/cache/add/all_adds_all_matching_versions_test.dart @@ -16,11 +16,13 @@ void main() { ..serve('foo', '2.0.0'); await runPub( - args: ['cache', 'add', 'foo', '-v', '>=1.0.0 <2.0.0', '--all'], - output: ''' - Downloading foo 1.2.2... - Downloading foo 1.2.3-dev... - Downloading foo 1.2.3...'''); + args: ['cache', 'add', 'foo', '-v', '>=1.0.0 <2.0.0', '--all'], + silent: allOf([ + contains('Downloading foo 1.2.2...'), + contains('Downloading foo 1.2.3-dev...'), + contains('Downloading foo 1.2.3...'), + ]), + ); await d.cacheDir({'foo': '1.2.2'}).validate(); await d.cacheDir({'foo': '1.2.3-dev'}).validate(); diff --git a/test/cache/add/all_with_some_versions_present_test.dart b/test/cache/add/all_with_some_versions_present_test.dart index 7b5b36d60..82d9fd584 100644 --- a/test/cache/add/all_with_some_versions_present_test.dart +++ b/test/cache/add/all_with_some_versions_present_test.dart @@ -18,18 +18,22 @@ void main() { // Install a couple of versions first. await runPub( args: ['cache', 'add', 'foo', '-v', '1.2.1'], - output: 'Downloading foo 1.2.1...'); + silent: contains('Downloading foo 1.2.1...')); await runPub( args: ['cache', 'add', 'foo', '-v', '1.2.3'], - output: 'Downloading foo 1.2.3...'); + silent: contains('Downloading foo 1.2.3...')); // They should show up as already installed now. - await runPub(args: ['cache', 'add', 'foo', '--all'], output: ''' - Already cached foo 1.2.1. - Downloading foo 1.2.2... - Already cached foo 1.2.3. - Downloading foo 2.0.0...'''); + await runPub( + args: ['cache', 'add', 'foo', '--all'], + silent: allOf([ + contains('Downloading foo 1.2.2...'), + contains('Downloading foo 2.0.0...') + ]), + output: ''' +Already cached foo 1.2.1. +Already cached foo 1.2.3.'''); await d.cacheDir({'foo': '1.2.1'}).validate(); await d.cacheDir({'foo': '1.2.2'}).validate(); diff --git a/test/cache/add/already_cached_test.dart b/test/cache/add/already_cached_test.dart index 8c74da9d4..ef9452e1e 100644 --- a/test/cache/add/already_cached_test.dart +++ b/test/cache/add/already_cached_test.dart @@ -14,7 +14,8 @@ void main() { // Run once to put it in the cache. await runPub( - args: ['cache', 'add', 'foo'], output: 'Downloading foo 1.2.3...'); + args: ['cache', 'add', 'foo'], + silent: contains('Downloading foo 1.2.3...')); // Should be in the cache now. await runPub( diff --git a/test/cache/repair/handles_failure_test.dart b/test/cache/repair/handles_failure_test.dart index d5637caf0..f44a42dbb 100644 --- a/test/cache/repair/handles_failure_test.dart +++ b/test/cache/repair/handles_failure_test.dart @@ -32,9 +32,6 @@ void main() { // Repair them. var pub = await startPub(args: ['cache', 'repair']); - expect(pub.stdout, emits('Downloading foo 1.2.3...')); - expect(pub.stdout, emits('Downloading foo 1.2.5...')); - expect(pub.stderr, emits(startsWith('Failed to repair foo 1.2.4. Error:'))); expect( pub.stderr, diff --git a/test/cache/repair/hosted.dart b/test/cache/repair/hosted.dart index ec3786be3..a826fad73 100644 --- a/test/cache/repair/hosted.dart +++ b/test/cache/repair/hosted.dart @@ -39,11 +39,12 @@ void main() { await runPub( args: ['cache', 'repair'], output: ''' - Downloading bar 1.2.4... - Downloading foo 1.2.3... - Downloading foo 1.2.5... + Reinstalled 3 packages.''', silent: allOf([ + contains('Downloading bar 1.2.4...'), + contains('Downloading foo 1.2.3...'), + contains('Downloading foo 1.2.5...'), contains('X-Pub-OS: ${Platform.operatingSystem}'), contains('X-Pub-Command: cache repair'), contains('X-Pub-Session-ID:'), diff --git a/test/cache/repair/recompiles_snapshots_test.dart b/test/cache/repair/recompiles_snapshots_test.dart index f3c3d6ca9..6fa4ca234 100644 --- a/test/cache/repair/recompiles_snapshots_test.dart +++ b/test/cache/repair/recompiles_snapshots_test.dart @@ -21,7 +21,6 @@ void main() { ]).create(); await runPub(args: ['cache', 'repair'], output: ''' - Downloading foo 1.0.0... Reinstalled 1 package. Reactivating foo 1.0.0... Building package executables... diff --git a/test/cache/repair/updates_binstubs_test.dart b/test/cache/repair/updates_binstubs_test.dart index e8cbfb483..65a574c04 100644 --- a/test/cache/repair/updates_binstubs_test.dart +++ b/test/cache/repair/updates_binstubs_test.dart @@ -34,7 +34,6 @@ void main() { // Repair them. await runPub(args: ['cache', 'repair'], output: ''' - Downloading foo 1.0.0... Reinstalled 1 package. Reactivating foo 1.0.0... Building package executables... diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 8a718bf23..6609a51d5 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -83,7 +83,7 @@ Future main() async { warning: allOf( contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( - 'Cache entry for foo-1.0.0 does not have content-hash matching pubspec.lock.'), + 'Content of foo-1.0.0 has changed compared to what was locked your pubspec.lock.'), ), exitCode: exit_codes.SUCCESS, ); @@ -110,14 +110,14 @@ Future main() async { await pubGet( warning: contains( - 'Content of foo-1.0.0 has changed compared to your previous pubspec.lock.', + 'Content of foo-1.0.0 has changed compared to what was locked your pubspec.lock.', ), exitCode: exit_codes.SUCCESS, ); final lockfile = loadYaml( File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final newHash = lockfile['packages']['foo']['description']['sha256']; - expect(newHash, server.getSha256('foo', '1.0.0')); + expect(newHash, await server.getSha256('foo', '1.0.0')); }); test( diff --git a/test/global/activate/activate_git_after_hosted_test.dart b/test/global/activate/activate_git_after_hosted_test.dart index 390aa8772..99f2fcb54 100644 --- a/test/global/activate/activate_git_after_hosted_test.dart +++ b/test/global/activate/activate_git_after_hosted_test.dart @@ -29,7 +29,7 @@ void main() { output: allOf( startsWith('Package foo is currently active at version 1.0.0.\n' 'Resolving dependencies...\n' - '+ foo 1.0.0 from git ..${separator}foo.git at '), + '* foo 1.0.0 from git ..${separator}foo.git at '), // Specific revision number goes here. endsWith('Building package executables...\n' 'Built foo:foo.\n' diff --git a/test/global/activate/activate_hosted_after_git_test.dart b/test/global/activate/activate_hosted_after_git_test.dart index 7ac174a85..8cd504464 100644 --- a/test/global/activate/activate_hosted_after_git_test.dart +++ b/test/global/activate/activate_hosted_after_git_test.dart @@ -25,8 +25,7 @@ void main() { await runPub(args: ['global', 'activate', 'foo'], output: ''' Package foo is currently active from Git repository "..${separator}foo.git". Resolving dependencies... - + foo 2.0.0 - Downloading foo 2.0.0... + * foo 2.0.0 (was 1.0.0 from git ../foo.git at 8162e8) Building package executables... Built foo:foo. Activated foo 2.0.0.'''); diff --git a/test/global/activate/activate_hosted_after_path_test.dart b/test/global/activate/activate_hosted_after_path_test.dart index 4cd753f09..d23ca472a 100644 --- a/test/global/activate/activate_hosted_after_path_test.dart +++ b/test/global/activate/activate_hosted_after_path_test.dart @@ -5,6 +5,7 @@ import 'package:path/path.dart' as p; import 'package:pub/src/io.dart'; import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart'; import '../../descriptor.dart' as d; import '../../test_pub.dart'; @@ -27,8 +28,7 @@ void main() { await runPub(args: ['global', 'activate', 'foo'], output: ''' Package foo is currently active at path "$path". Resolving dependencies... - + foo 2.0.0 - Downloading foo 2.0.0... + * foo 2.0.0 (was 1.0.0 from path $path) Building package executables... Built foo:foo. Activated foo 2.0.0.'''); diff --git a/test/global/activate/activate_hosted_twice_test.dart b/test/global/activate/activate_hosted_twice_test.dart index a4d1337ef..fb4ce928e 100644 --- a/test/global/activate/activate_hosted_twice_test.dart +++ b/test/global/activate/activate_hosted_twice_test.dart @@ -46,9 +46,7 @@ Activated foo 1.0.0.'''); await runPub(args: ['global', 'activate', 'foo'], output: ''' Package foo is currently active at version 1.0.0. Resolving dependencies... -+ bar 2.0.0 -+ foo 1.0.0 -Downloading bar 2.0.0... +> bar 2.0.0 (was 1.0.0) Building package executables... Built foo:foo. Activated foo 1.0.0.'''); diff --git a/test/global/activate/custom_hosted_url_test.dart b/test/global/activate/custom_hosted_url_test.dart index 8b2f0fc34..54a1b7947 100644 --- a/test/global/activate/custom_hosted_url_test.dart +++ b/test/global/activate/custom_hosted_url_test.dart @@ -26,12 +26,13 @@ void main() { customServer.serve('bar', '1.0.0', deps: {'baz': 'any'}); await runPub( - args: ['global', 'activate', 'foo', '-u', customServer.url], - output: allOf([ - contains('Downloading bar 1.0.0...'), - contains('Downloading baz 1.0.0...'), - contains('Downloading foo 1.0.0...'), - contains('Activated foo 1.0.0') - ])); + args: ['global', 'activate', 'foo', '-u', customServer.url], + silent: allOf([ + contains('Downloading bar 1.0.0...'), + contains('Downloading baz 1.0.0...'), + contains('Downloading foo 1.0.0...'), + ]), + output: contains('Activated foo 1.0.0'), + ); }); } diff --git a/test/global/activate/different_version_test.dart b/test/global/activate/different_version_test.dart index 8406b9269..d84ed8cbf 100644 --- a/test/global/activate/different_version_test.dart +++ b/test/global/activate/different_version_test.dart @@ -26,8 +26,7 @@ void main() { await runPub(args: ['global', 'activate', 'foo', '>1.0.0'], output: ''' Package foo is currently active at version 1.0.0. Resolving dependencies... - + foo 2.0.0 - Downloading foo 2.0.0... + > foo 2.0.0 (was 1.0.0) Building package executables... Built foo:foo. Activated foo 2.0.0.'''); diff --git a/test/global/activate/ignores_active_version_test.dart b/test/global/activate/ignores_active_version_test.dart index 777a63127..d3e609498 100644 --- a/test/global/activate/ignores_active_version_test.dart +++ b/test/global/activate/ignores_active_version_test.dart @@ -25,8 +25,7 @@ void main() { await runPub(args: ['global', 'activate', 'foo', '>1.0.0'], output: ''' Package foo is currently active at version 1.2.3. Resolving dependencies... - + foo 1.3.0 - Downloading foo 1.3.0... + > foo 1.3.0 (was 1.2.3) Building package executables... Built foo:foo. Activated foo 1.3.0.'''); diff --git a/test/global/activate/installs_dependencies_for_git_test.dart b/test/global/activate/installs_dependencies_for_git_test.dart index 6018f5061..04bdc5364 100644 --- a/test/global/activate/installs_dependencies_for_git_test.dart +++ b/test/global/activate/installs_dependencies_for_git_test.dart @@ -20,7 +20,7 @@ void main() { await runPub( args: ['global', 'activate', '-sgit', '../foo.git'], - output: allOf([ + silent: allOf([ contains('Downloading bar 1.0.0...'), contains('Downloading baz 1.0.0...') ])); diff --git a/test/global/activate/installs_dependencies_for_path_test.dart b/test/global/activate/installs_dependencies_for_path_test.dart index 4688f0749..ae9024244 100644 --- a/test/global/activate/installs_dependencies_for_path_test.dart +++ b/test/global/activate/installs_dependencies_for_path_test.dart @@ -20,8 +20,6 @@ void main() { var pub = await startPub(args: ['global', 'activate', '-spath', '../foo']); expect(pub.stdout, emitsThrough('Resolving dependencies in ../foo...')); - expect(pub.stdout, emitsThrough('Downloading bar 1.0.0...')); - expect(pub.stdout, emitsThrough('Downloading baz 2.0.0...')); expect(pub.stdout, emitsThrough(startsWith('Activated foo 0.0.0 at path'))); await pub.shouldExit(); diff --git a/test/global/activate/installs_dependencies_test.dart b/test/global/activate/installs_dependencies_test.dart index 770ebcfa9..9cb235480 100644 --- a/test/global/activate/installs_dependencies_test.dart +++ b/test/global/activate/installs_dependencies_test.dart @@ -15,7 +15,7 @@ void main() { await runPub( args: ['global', 'activate', 'foo'], - output: allOf([ + silent: allOf([ contains('Downloading bar 1.0.0...'), contains('Downloading baz 1.0.0...') ])); diff --git a/test/global/activate/reactivating_git_upgrades_test.dart b/test/global/activate/reactivating_git_upgrades_test.dart index 5eb8d5f88..6ce628de3 100644 --- a/test/global/activate/reactivating_git_upgrades_test.dart +++ b/test/global/activate/reactivating_git_upgrades_test.dart @@ -36,7 +36,7 @@ void main() { startsWith('Package foo is currently active from Git repository ' '"..${separator}foo.git".\n' 'Resolving dependencies...\n' - '+ foo 1.0.1 from git ..${separator}foo.git at '), + '> foo 1.0.1 from git ..${separator}foo.git at '), // Specific revision number goes here. endsWith('Building package executables...\n' 'Built foo:foo.\n' diff --git a/test/global/activate/uncached_package_test.dart b/test/global/activate/uncached_package_test.dart index 1a18d70d5..cb8d26de7 100644 --- a/test/global/activate/uncached_package_test.dart +++ b/test/global/activate/uncached_package_test.dart @@ -23,7 +23,6 @@ void main() { await runPub(args: ['global', 'activate', 'foo'], output: ''' Resolving dependencies... + foo 1.2.3 - Downloading foo 1.2.3... Building package executables... Built foo:foo. Activated foo 1.2.3.'''); diff --git a/test/global/deactivate/deactivate_and_reactivate_package_test.dart b/test/global/deactivate/deactivate_and_reactivate_package_test.dart index da3348647..5a6f6e34d 100644 --- a/test/global/deactivate/deactivate_and_reactivate_package_test.dart +++ b/test/global/deactivate/deactivate_and_reactivate_package_test.dart @@ -20,10 +20,12 @@ void main() { output: 'Deactivated package foo 1.0.0.'); // Activating again should forget the old version. - await runPub(args: ['global', 'activate', 'foo'], output: ''' + await runPub( + args: ['global', 'activate', 'foo'], + silent: contains('Downloading foo 2.0.0...'), + output: ''' Resolving dependencies... + foo 2.0.0 - Downloading foo 2.0.0... Activated foo 2.0.0.'''); }); } diff --git a/test/test_pub.dart b/test/test_pub.dart index b542e180b..7d30b6961 100644 --- a/test/test_pub.dart +++ b/test/test_pub.dart @@ -206,6 +206,7 @@ Future pubUpgrade( Object? output, Object? error, Object? warning, + Object? silent, int? exitCode, Map? environment, String? workingDirectory}) async => @@ -215,6 +216,7 @@ Future pubUpgrade( output: output, error: error, warning: warning, + silent: silent, exitCode: exitCode, environment: environment, workingDirectory: workingDirectory, @@ -862,9 +864,6 @@ StreamMatcher emitsLines(String output) => emitsInOrder(output.split('\n')); /// Removes output from pub known to be unstable. Iterable filterUnstableLines(List input) { return input - // Downloading order is not deterministic, so to avoid flakiness we filter - // out these lines. - .where((line) => !line.startsWith('Downloading ')) // Any paths in output should be relative to the sandbox and with forward // slashes to be stable across platforms. .map((line) { diff --git a/test/testdata/goldens/embedding/embedding_test/--color forces colors.txt b/test/testdata/goldens/embedding/embedding_test/--color forces colors.txt index d468889b0..e4ad1d17c 100644 --- a/test/testdata/goldens/embedding/embedding_test/--color forces colors.txt +++ b/test/testdata/goldens/embedding/embedding_test/--color forces colors.txt @@ -3,7 +3,6 @@ $ tool/test-bin/pub_command_runner.dart pub --no-color get Resolving dependencies... + foo 1.0.0 (2.0.0 available) -Downloading foo 1.0.0... Changed 1 dependency! -------------------------------- END OF OUTPUT --------------------------------- diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 05c13bc93..5e62afa73 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -3,7 +3,6 @@ $ tool/test-bin/pub_command_runner.dart pub --verbose get MSG : Resolving dependencies... MSG : + foo 1.0.0 -MSG : Downloading foo 1.0.0... MSG : Changed 1 dependency! MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] FINE: Pub 0.1.2+3 @@ -38,6 +37,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | Tried 1 solutions. [E] FINE: Resolving dependencies finished ($TIME) [E] IO : Get package from http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz. +[E] FINE: Downloading foo 1.0.0... [E] IO : Created temp directory $DIR [E] IO : HTTP GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz [E] | X-Pub-OS: $OS @@ -178,9 +178,8 @@ SLVR: selecting foo 1.0.0 SLVR: Version solving took: $TIME | Tried 1 solutions. FINE: Resolving dependencies finished ($TIME) -MSG : + foo 1.0.0 IO : Get package from http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz. -MSG : Downloading foo 1.0.0... +FINE: Downloading foo 1.0.0... IO : Created temp directory $DIR IO : HTTP GET http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz | X-Pub-OS: $OS @@ -213,6 +212,7 @@ FINE: Created $FILE from stream FINE: Extracted .tar.gz to $DIR IO : Renaming directory $A to $B IO : Deleting directory $DIR +MSG : + foo 1.0.0 IO : Writing $N characters to text file pubspec.lock. FINE: Contents: | # Generated by pub diff --git a/test/upgrade/dry_run_does_not_apply_changes_test.dart b/test/upgrade/dry_run_does_not_apply_changes_test.dart index 8c942ca79..4f514f998 100644 --- a/test/upgrade/dry_run_does_not_apply_changes_test.dart +++ b/test/upgrade/dry_run_does_not_apply_changes_test.dart @@ -68,6 +68,7 @@ void main() { // Do the dry run. await pubUpgrade( args: ['--dry-run', '--major-versions'], + silent: contains('Downloading foo 2.0.0...'), output: allOf([ contains('Resolving dependencies...'), contains('> foo 2.0.0 (was 1.0.0)'), @@ -92,7 +93,6 @@ void main() { output: allOf([ contains('Resolving dependencies...'), contains('> foo 2.0.0 (was 1.0.0)'), - contains('Downloading foo 2.0.0...'), contains('Changed 1 dependency!'), contains('Changed 1 constraint in pubspec.yaml:'), contains('foo: ^1.0.0 -> ^2.0.0'), diff --git a/tool/test.dart b/tool/test.dart index 57ab55464..e0ef4b073 100755 --- a/tool/test.dart +++ b/tool/test.dart @@ -39,7 +39,7 @@ Future main(List args) async { packageConfigPath: path.join('.dart_tool', 'package_config.json')); testProcess = await Process.start( Platform.resolvedExecutable, - ['run', 'test', '--chain-stack-traces', ...args], + ['run', 'test', ...args], environment: {'_PUB_TEST_SNAPSHOT': pubSnapshotFilename}, mode: ProcessStartMode.inheritStdio, ); From 11cf1d23e8d728eb8b21e345b3adcf6405e943cb Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 1 Sep 2022 11:41:50 +0000 Subject: [PATCH 17/51] lint --- lib/src/source/hosted.dart | 11 ++--- test/content_hash_test.dart | 42 +++++++------------ test/descriptor.dart | 14 +++++++ .../activate_hosted_after_path_test.dart | 1 - test/package_server.dart | 3 ++ ...--verbose and on unexpected exceptions.txt | 4 +- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 113ef8997..9d00e8e3a 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -767,8 +767,8 @@ class HostedSource extends CachedSource { /// The system cache directory for the hosted source contains subdirectories /// for each separate repository URL that's used on the system. /// - /// Each of these subdirectories then contains a `.hashes` directory with a - /// stored hash of all downloaded packages. + /// Parallel to this there is a `hosted-hashes` directory with a stored hash + /// of all downloaded packages. String hashPath(PackageId id, SystemCache cache) { final description = id.description.description; if (description is! HostedDescription) { @@ -776,8 +776,9 @@ class HostedSource extends CachedSource { } final rootDir = cache.rootDirForSource(this); - var dir = _urlToDirectory(description.url); - return p.join(rootDir, dir, '.hashes', '${id.name}-${id.version}.sha256'); + var serverDir = _urlToDirectory(description.url); + return p.join(rootDir, 'hosted-hashes', serverDir, '.hashes', + '${id.name}-${id.version}.sha256'); } /// Loads the hash at `hashPath(id)`. @@ -1162,7 +1163,7 @@ class ResolvedHostedDescription extends ResolvedDescription { /// (will be null if the server does not report this.) /// * Obtained from a pubspec.lock /// (will be null for legacy lock-files). - /// * Read from the /hosted/.hashes/-.sha256 file. + /// * Read from the /hosted-hashes//-.sha256 file. /// (will be null if the file doesn't exist for corrupt or legacy caches). final Uint8List? sha256; diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 6609a51d5..a86d41850 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -25,10 +25,8 @@ Future main() async { File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final sha256 = lockfile['packages']['foo']['description']['sha256']; expect(sha256, hasLength(64)); - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', sha256), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', sha256), ]).validate(); }); @@ -44,10 +42,8 @@ Future main() async { File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final sha256 = lockfile['packages']['foo']['description']['sha256']; expect(sha256, hasLength(64)); - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', sha256), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', sha256), ]).validate(); }); @@ -105,7 +101,7 @@ Future main() async { contents: [file('new_file.txt', 'This file could be malicious.')]); // Deleting the hash-file cache will cause it to be refetched, and the // warning will happen. - File(p.join(globalServer.cachingPath, '.hashes', 'foo-1.0.0.sha256')) + File(p.join(globalServer.hashesCachingPath, '.hashes', 'foo-1.0.0.sha256')) .deleteSync(); await pubGet( @@ -132,19 +128,15 @@ Future main() async { File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final originalHash = lockfile['packages']['foo']['description']['sha256']; // Create wrong hash on disk. - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', - 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', + 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), ]).create(); await pubGet( warning: 'Cached version of foo-1.0.0 has wrong hash - redownloading.'); - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', originalHash), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', originalHash), ]).validate(); }); @@ -158,19 +150,15 @@ Future main() async { final lockfile = loadYaml( File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final originalHash = lockfile['packages']['foo']['description']['sha256']; - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', - 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', + 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'), ]).create(); await pubGet( warning: 'Cached version of foo-1.0.0 has wrong hash - redownloading.'); - await hostedCache([ - dir('.hashes', [ - file('foo-1.0.0.sha256', originalHash), - ]) + await hostedHashesCache([ + file('foo-1.0.0.sha256', originalHash), ]).validate(); }); test( diff --git a/test/descriptor.dart b/test/descriptor.dart index 9248ad471..fd7b85dc5 100644 --- a/test/descriptor.dart +++ b/test/descriptor.dart @@ -193,6 +193,20 @@ Descriptor hostedCache(Iterable contents, {int? port}) { ]); } +/// Describes the hosted-hashes cache directory containing hashes of the hosted +/// packages downloaded from the mock package server. +/// +/// If [port] is passed, it's used as the port number of the local hosted server +/// that this cache represents. It defaults to [globalServer.port]. +Descriptor hostedHashesCache(Iterable contents, {int? port}) { + return dir(cachePath, [ + dir( + 'hosted-hashes', + [dir('localhost%58${port ?? globalServer.port}', contents)], + ) + ]); +} + /// Describes the file that contains the client's OAuth2 /// credentials. The URL "/token" on [server] will be used as the token /// endpoint for refreshing the access token. diff --git a/test/global/activate/activate_hosted_after_path_test.dart b/test/global/activate/activate_hosted_after_path_test.dart index d23ca472a..8f91a1f7e 100644 --- a/test/global/activate/activate_hosted_after_path_test.dart +++ b/test/global/activate/activate_hosted_after_path_test.dart @@ -5,7 +5,6 @@ import 'package:path/path.dart' as p; import 'package:pub/src/io.dart'; import 'package:test/test.dart'; -import 'package:test_descriptor/test_descriptor.dart'; import '../../descriptor.dart' as d; import '../../test_pub.dart'; diff --git a/test/package_server.dart b/test/package_server.dart index b2d42bcbf..ef4f1c7e8 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -178,6 +178,9 @@ class PackageServer { String get cachingPath => p.join(d.sandbox, cachePath, 'hosted', 'localhost%58$port'); + String get hashesCachingPath => + p.join(d.sandbox, cachePath, 'hosted-hashes', 'localhost%58$port'); + /// A map from package names to the concrete packages to serve. final _packages = {}; diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 5e62afa73..4ca1bd43b 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -56,7 +56,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff [E] IO : Creating $FILE from stream -[E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +[E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/hosted-hashes/localhost%58$PORT/.hashes/foo-1.0.0.sha256. [E] FINE: Contents: [E] | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb [E] FINE: Created $FILE from stream @@ -198,7 +198,7 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/ | x-xss-protection: 1; mode=block | x-content-type-options: nosniff IO : Creating $FILE from stream -IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +IO : Writing $N characters to text file $SANDBOX/cache/hosted/hosted-hashes/localhost%58$PORT/.hashes/foo-1.0.0.sha256. FINE: Contents: | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb FINE: Created $FILE from stream From fe90a681bdce2e47a066107efeb8307eab24cb4a Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 1 Sep 2022 12:05:35 +0000 Subject: [PATCH 18/51] fixes --- lib/src/source/hosted.dart | 6 +++--- test/content_hash_test.dart | 2 +- ... written with --verbose and on unexpected exceptions.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 9d00e8e3a..3bb5b6c48 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -774,11 +774,11 @@ class HostedSource extends CachedSource { if (description is! HostedDescription) { throw ArgumentError('Wrong source'); } - final rootDir = cache.rootDirForSource(this); + final rootDir = cache.rootDir; var serverDir = _urlToDirectory(description.url); - return p.join(rootDir, 'hosted-hashes', serverDir, '.hashes', - '${id.name}-${id.version}.sha256'); + return p.join( + rootDir, 'hosted-hashes', serverDir, '${id.name}-${id.version}.sha256'); } /// Loads the hash at `hashPath(id)`. diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index a86d41850..d0860d764 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -101,7 +101,7 @@ Future main() async { contents: [file('new_file.txt', 'This file could be malicious.')]); // Deleting the hash-file cache will cause it to be refetched, and the // warning will happen. - File(p.join(globalServer.hashesCachingPath, '.hashes', 'foo-1.0.0.sha256')) + File(p.join(globalServer.hashesCachingPath, 'foo-1.0.0.sha256')) .deleteSync(); await pubGet( diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 4ca1bd43b..8d2d14d1f 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -56,7 +56,7 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff [E] IO : Creating $FILE from stream -[E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/hosted-hashes/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +[E] IO : Writing $N characters to text file $SANDBOX/cache/hosted-hashes/localhost%58$PORT/foo-1.0.0.sha256. [E] FINE: Contents: [E] | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb [E] FINE: Created $FILE from stream @@ -198,7 +198,7 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/packages/foo/versions/ | x-xss-protection: 1; mode=block | x-content-type-options: nosniff IO : Creating $FILE from stream -IO : Writing $N characters to text file $SANDBOX/cache/hosted/hosted-hashes/localhost%58$PORT/.hashes/foo-1.0.0.sha256. +IO : Writing $N characters to text file $SANDBOX/cache/hosted-hashes/localhost%58$PORT/foo-1.0.0.sha256. FINE: Contents: | 439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb FINE: Created $FILE from stream From ab0e5a4fc04320582e7da4f840353f6b5e49608c Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 1 Sep 2022 12:17:36 +0000 Subject: [PATCH 19/51] Document SystemCache.downloadPackage --- lib/src/system_cache.dart | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart index 3fe617e1a..4b3ee43da 100644 --- a/lib/src/system_cache.dart +++ b/lib/src/system_cache.dart @@ -213,6 +213,14 @@ class SystemCache { return id.source.doGetDirectory(id, this, relativeFrom: relativeFrom); } + /// Downloads a cached package identified by [id] to the cache. + /// + /// [id] must refer to a cached package. + /// + /// If [allowOutdatedHashChecks] is `true` we use a cached version listing + /// response if present instead of probing the server. Not probing allows for + /// `pub get` with a filled cache to be a fast case that doesn't require any + /// new version-listings. Future downloadPackage( PackageId id, { required bool allowOutdatedHashChecks, From 72634e7b6a4cf218ec12cd882350b012a0a32cfb Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 1 Sep 2022 12:44:22 +0000 Subject: [PATCH 20/51] test package server hash content-hashes by default --- test/package_server.dart | 4 ++-- .../Relative paths are allowed.txt | 6 ++++-- ...ritten with --verbose and on unexpected exceptions.txt | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/package_server.dart b/test/package_server.dart index ef4f1c7e8..835b842d6 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -29,8 +29,8 @@ class PackageServer { // A list of all the requests received up till now. final List requestedPaths = []; - // Setting this to true will make automatic calculation of content-hashes. - bool serveContentHashes = false; + // Setting this to false will disable automatic calculation of content-hashes. + bool serveContentHashes = true; PackageServer._(this._inner) { _inner.mount((request) { diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt index 82badd7c1..2e3877339 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt @@ -106,7 +106,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0" } }, "constraintBumped": "^2.0.0", @@ -133,7 +134,8 @@ $ dependency_services report "type": "hosted", "description": { "name": "foo", - "url": "http://localhost:$PORT" + "url": "http://localhost:$PORT", + "sha256": "c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0" } }, "constraintBumped": "^2.0.0", diff --git a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt index 8d2d14d1f..5ba53987e 100644 --- a/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt +++ b/test/testdata/goldens/embedding/embedding_test/logfile is written with --verbose and on unexpected exceptions.txt @@ -24,14 +24,14 @@ MSG : Logs written to $SANDBOX/cache/log/pub_log.txt. [E] | took: $TIME [E] | x-powered-by: Dart with package:shelf [E] | date: $TIME -[E] | content-length: 197 +[E] | content-length: 281 [E] | x-frame-options: SAMEORIGIN [E] | content-type: text/plain; charset=utf-8 [E] | x-xss-protection: 1; mode=block [E] | x-content-type-options: nosniff [E] IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. [E] FINE: Contents: -[E] | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz"}],"_fetchedAt": "$TIME"} +[E] | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz","archive_sha256":"439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb"}],"_fetchedAt": "$TIME"} [E] SLVR: selecting foo 1.0.0 [E] SLVR: Version solving took: $TIME [E] | Tried 1 solutions. @@ -166,14 +166,14 @@ IO : HTTP response 200 OK for GET http://localhost:$PORT/api/packages/foo | took: $TIME | x-powered-by: Dart with package:shelf | date: $TIME - | content-length: 197 + | content-length: 281 | x-frame-options: SAMEORIGIN | content-type: text/plain; charset=utf-8 | x-xss-protection: 1; mode=block | x-content-type-options: nosniff IO : Writing $N characters to text file $SANDBOX/cache/hosted/localhost%58$PORT/.cache/foo-versions.json. FINE: Contents: - | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz"}],"_fetchedAt": "$TIME"} + | {"name":"foo","uploaders":["nweiz@google.com"],"versions":[{"pubspec":{"name":"foo","version":"1.0.0"},"version":"1.0.0","archive_url":"http://localhost:$PORT/packages/foo/versions/1.0.0.tar.gz","archive_sha256":"439814f59cbc73e1c28ca5ac6e437d5f2af10dfd18db786ce46fe0663e605ccb"}],"_fetchedAt": "$TIME"} SLVR: selecting foo 1.0.0 SLVR: Version solving took: $TIME | Tried 1 solutions. From ec54824a0af6a03e8c259f64ed61d290d6be07e1 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 09:17:39 +0000 Subject: [PATCH 21/51] Fix separator for windows in test --- test/global/activate/activate_hosted_after_git_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/global/activate/activate_hosted_after_git_test.dart b/test/global/activate/activate_hosted_after_git_test.dart index 8cd504464..e63152f6e 100644 --- a/test/global/activate/activate_hosted_after_git_test.dart +++ b/test/global/activate/activate_hosted_after_git_test.dart @@ -25,7 +25,7 @@ void main() { await runPub(args: ['global', 'activate', 'foo'], output: ''' Package foo is currently active from Git repository "..${separator}foo.git". Resolving dependencies... - * foo 2.0.0 (was 1.0.0 from git ../foo.git at 8162e8) + * foo 2.0.0 (was 1.0.0 from git ..${separator}foo.git at 8162e8) Building package executables... Built foo:foo. Activated foo 2.0.0.'''); From 1b1765db23cfd17d720c6b85d4f7dc1393ad6031 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 09:48:18 +0000 Subject: [PATCH 22/51] Test of package-change-prefixes --- .../upgrade/report/describes_change_test.dart | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/test/upgrade/report/describes_change_test.dart b/test/upgrade/report/describes_change_test.dart index 96410cb67..2f926183e 100644 --- a/test/upgrade/report/describes_change_test.dart +++ b/test/upgrade/report/describes_change_test.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:path/path.dart'; import 'package:test/test.dart'; import '../../descriptor.dart' as d; @@ -33,11 +34,17 @@ void main() { }); test('shows how package changed from previous lockfile', () async { - await servePackages() - ..serve('unchanged', '1.0.0') - ..serve('version_changed', '1.0.0') - ..serve('version_changed', '2.0.0') - ..serve('source_changed', '1.0.0'); + final server = await servePackages(); + + server.serve('unchanged', '1.0.0'); + server.serve('version_upgraded', '1.0.0'); + server.serve('version_upgraded', '2.0.0'); + server.serve('version_downgraded', '1.0.0'); + server.serve('version_downgraded', '2.0.0'); + server.serve('contents_changed', '1.0.0'); + server.serve('source_changed', '1.0.0'); + server.serve('package_added', '1.0.0'); + server.serve('package_removed', '1.0.0'); await d.dir('source_changed', [ d.libDir('source_changed'), @@ -57,28 +64,47 @@ void main() { // Create the first lockfile. await d.appDir({ 'unchanged': 'any', - 'version_changed': '1.0.0', + 'contents_changed': '1.0.0', + 'version_upgraded': '1.0.0', + 'version_downgraded': '2.0.0', 'source_changed': 'any', + 'package_removed': 'any', 'description_changed': {'path': '../description_changed_1'} }).create(); await pubGet(); + server.serve( + 'contents_changed', + '1.0.0', + contents: [d.file('Sneaky.txt', 'Very sneaky attack on integrity.')], + ); // Change the pubspec. await d.appDir({ 'unchanged': 'any', - 'version_changed': 'any', + 'version_upgraded': 'any', + 'version_downgraded': '1.0.0', 'source_changed': {'path': '../source_changed'}, - 'description_changed': {'path': '../description_changed_2'} + 'package_added': 'any', + 'description_changed': {'path': '../description_changed_2'}, + 'contents_changed': '1.0.0', }).create(); // Upgrade everything. - await pubUpgrade(output: RegExp(r''' -Resolving dependencies\.\.\..* -. description_changed 1\.0\.0 from path \.\.[/\\]description_changed_2 \(was 1\.0\.0 from path \.\.[/\\]description_changed_1\) -. source_changed 2\.0\.0 from path \.\.[/\\]source_changed \(was 1\.0\.0\) -. unchanged 1\.0\.0 -. version_changed 2\.0\.0 \(was 1\.0\.0\) -''', multiLine: true), environment: {'PUB_ALLOW_PRERELEASE_SDK': 'false'}); + await pubUpgrade( + output: allOf([ + contains('Resolving dependencies...'), + contains( + '* description_changed 1.0.0 from path ..${separator}description_changed_2 (was 1.0.0 from path ..${separator}description_changed_1)'), + contains(' unchanged 1.0.0'), + contains( + '* source_changed 2.0.0 from path ..${separator}source_changed (was 1.0.0)'), + contains('> version_upgraded 2.0.0 (was 1.0.0'), + contains('< version_downgraded 1.0.0 (was 2.0.0'), + contains('+ package_added 1.0.0'), + contains('- package_removed 1.0.0'), + contains('~ contents_changed 1.0.0 (was 1.0.0)'), + ]), + environment: {'PUB_ALLOW_PRERELEASE_SDK': 'false'}); }); } From 80f56cdbff12f78c5c5e8c07103367fdf7799d28 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 11:33:19 +0000 Subject: [PATCH 23/51] Link to spec in gzip testing hack --- test/package_server.dart | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/package_server.dart b/test/package_server.dart index 835b842d6..ae968d6eb 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -243,8 +243,12 @@ class PackageServer { ); } - /// Replaces the 9th entry in [stream] with a 0. This replaces the os entry - /// of a gzip stream, giving us the same stream on all platforms. + /// Replaces the 9th entry in [stream] with a 0. This replaces the os entry of + /// a gzip stream, giving us the same stream and thius stable testing on all + /// platforms. + /// + /// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information + /// about the OS header. Stream> replaceOs(Stream> stream) async* { var i = 0; await for (final t in stream) { From 040abefa91fdb9218a14c68cf7827aaecabb85a6 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 11:34:12 +0000 Subject: [PATCH 24/51] Make replaceOs private --- test/package_server.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/package_server.dart b/test/package_server.dart index ae968d6eb..b0abde969 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -236,7 +236,7 @@ class PackageServer { for (final e in contents ?? []) { addDescriptor(e, ''); } - return replaceOs(Stream.fromIterable(entries) + return _replaceOs(Stream.fromIterable(entries) .transform(tarWriterWith(format: OutputFormat.gnuLongName)) .transform(gzip.encoder)); }, @@ -249,7 +249,7 @@ class PackageServer { /// /// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information /// about the OS header. - Stream> replaceOs(Stream> stream) async* { + Stream> _replaceOs(Stream> stream) async* { var i = 0; await for (final t in stream) { if (i > 9 || (i + t.length < 9)) { From 660eb215c8f5cab95356568f4d9325804893e0a1 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 11:43:21 +0000 Subject: [PATCH 25/51] Simplify gzip hack --- test/package_server.dart | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/test/package_server.dart b/test/package_server.dart index b0abde969..2cba860c6 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -4,7 +4,8 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:io'; +import 'dart:io' hide BytesBuilder; +import 'dart:typed_data' show BytesBuilder; import 'package:crypto/crypto.dart'; import 'package:path/path.dart' as p; @@ -243,25 +244,20 @@ class PackageServer { ); } - /// Replaces the 9th entry in [stream] with a 0. This replaces the os entry of - /// a gzip stream, giving us the same stream and thius stable testing on all - /// platforms. + /// Replaces the entry at index 9 in [stream] with a 0. This replaces the os + /// entry of a gzip stream, giving us the same stream and thius stable testing + /// on all platforms. /// /// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information /// about the OS header. Stream> _replaceOs(Stream> stream) async* { - var i = 0; + final bytesBuilder = BytesBuilder(); await for (final t in stream) { - if (i > 9 || (i + t.length < 9)) { - yield t; - i += t.length; - continue; - } - yield t.sublist(0, 9 - i); - yield [0]; - yield t.sublist(9 - i + 1); - i += t.length; + bytesBuilder.add(t); } + final result = bytesBuilder.toBytes(); + result[9] = 0; + yield result; } // Mark a package discontinued. From e99ede5f9df6bf47c7b024961430a7729d13abe8 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 2 Sep 2022 11:48:07 +0000 Subject: [PATCH 26/51] setSha256 => overrideArchiveSha256 --- test/content_hash_test.dart | 6 +++--- test/package_server.dart | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index d0860d764..5c2d5f7c0 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -50,7 +50,7 @@ Future main() async { test('archive_sha256 is checked on download', () async { final server = await servePackages(); server.serve('foo', '1.0.0'); - server.setSha256('foo', '1.0.0', + server.overrideArchiveSha256('foo', '1.0.0', 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'); await appDir({'foo': 'any'}).create(); await pubGet( @@ -86,7 +86,7 @@ Future main() async { final lockfile = loadYaml( File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final newHash = lockfile['packages']['foo']['description']['sha256']; - expect(newHash, await server.getSha256('foo', '1.0.0')); + expect(newHash, await server.peekArchiveSha256('foo', '1.0.0')); }); test( @@ -113,7 +113,7 @@ Future main() async { final lockfile = loadYaml( File(p.join(sandbox, appPath, 'pubspec.lock')).readAsStringSync()); final newHash = lockfile['packages']['foo']['description']['sha256']; - expect(newHash, await server.getSha256('foo', '1.0.0')); + expect(newHash, await server.peekArchiveSha256('foo', '1.0.0')); }); test( diff --git a/test/package_server.dart b/test/package_server.dart index 2cba860c6..c534bfe1d 100644 --- a/test/package_server.dart +++ b/test/package_server.dart @@ -278,11 +278,11 @@ class PackageServer { } /// Useful for testing handling of a wrong hash. - void setSha256(String name, String version, String sha256) { + void overrideArchiveSha256(String name, String version, String sha256) { _packages[name]!.versions[version]!.sha256 = sha256; } - Future getSha256(String name, String version) async { + Future peekArchiveSha256(String name, String version) async { final v = _packages[name]!.versions[version]!; return v.sha256 ?? hexEncode((await sha256.bind(v.contents()).first).bytes); } From fdf1e76bb4fba36db4fbf7420f4a1efa8e0b9261 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 23 Sep 2022 11:57:22 +0200 Subject: [PATCH 27/51] Update doc/repository-spec-v2.md --- doc/repository-spec-v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/repository-spec-v2.md b/doc/repository-spec-v2.md index 57398472f..c5e226067 100644 --- a/doc/repository-spec-v2.md +++ b/doc/repository-spec-v2.md @@ -258,7 +258,7 @@ parameters. This allows for the server to return signed-URLs for S3, GCS or other blob storage service. If temporary URLs are returned it is wise to not set expiration to less than 25 minutes (to allow for retries and clock drift). -The `archive_sha256` should be the hex-encoded sha256 checksum of the file are +The `archive_sha256` should be the hex-encoded sha256 checksum of the file at archive_url. It is an optional field that allows the pub client to verify the integrity of downloaded archive. From 76107a6116730133c608edbbf9e67f354c9a6572 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 27 Sep 2022 15:44:05 +0200 Subject: [PATCH 28/51] Update lib/src/solver/result.dart --- lib/src/solver/result.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 88812217c..099ee8188 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -114,7 +114,7 @@ class SolveResult { if (originalHash != null) { if (!bytesEquals(cachedHash, originalHash)) { log.warning(''' -Content of ${id.name}-${id.version} has changed compared to what was locked your pubspec.lock. +The content of ${id.name}-${id.version} on the server doesn't match what was locked in your pubspec.lock. This might indicate that: * The content has changed on the server since you created the pubspec.lock. From 84c505140bc2ca0e83992f9484102bb236655235 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 27 Sep 2022 15:44:38 +0200 Subject: [PATCH 29/51] Update lib/src/solver/result.dart --- lib/src/solver/result.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 099ee8188..2ac3939ab 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -120,7 +120,7 @@ This might indicate that: * The content has changed on the server since you created the pubspec.lock. * The pubspec.lock has been corrupted. -Your pubspec.lock has been updated according to what is on your server. +Your pubspec.lock has been updated according to what is on the server. See $contentHashesDocumentationUrl for more information. '''); From 514284616378e19658ee17715b79e6ecf584a0e0 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 27 Sep 2022 13:46:01 +0000 Subject: [PATCH 30/51] Typo --- doc/repository-spec-v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/repository-spec-v2.md b/doc/repository-spec-v2.md index 3dc33c17f..c4e705039 100644 --- a/doc/repository-spec-v2.md +++ b/doc/repository-spec-v2.md @@ -260,7 +260,7 @@ expiration to less than 25 minutes (to allow for retries and clock drift). The `archive_sha256` should be the hex-encoded sha256 checksum of the file are archive_url. It is an optional field that allows the pub client to verify the -integrity of downloaded archive. +integrity of the downloaded archive. The `archive_sha256` also provides an easy way for clients to detect if something has changed on the server. In the absense of this field the client can From 0dcb2df63c7636b0136860971eceb6a26ea2278d Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 27 Sep 2022 13:46:39 +0000 Subject: [PATCH 31/51] Fix test --- .../activate_hosted_after_git_test.dart | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/global/activate/activate_hosted_after_git_test.dart b/test/global/activate/activate_hosted_after_git_test.dart index e63152f6e..e3c5a455c 100644 --- a/test/global/activate/activate_hosted_after_git_test.dart +++ b/test/global/activate/activate_hosted_after_git_test.dart @@ -22,13 +22,20 @@ void main() { await runPub(args: ['global', 'activate', '-sgit', '../foo.git']); - await runPub(args: ['global', 'activate', 'foo'], output: ''' - Package foo is currently active from Git repository "..${separator}foo.git". - Resolving dependencies... - * foo 2.0.0 (was 1.0.0 from git ..${separator}foo.git at 8162e8) - Building package executables... - Built foo:foo. - Activated foo 2.0.0.'''); + await runPub(args: ['global', 'activate', 'foo'], output: + predicate((output) { + return (output as String).replaceFirst(RegExp(' [a-f0-9]{5}'), ' ....') == ''' +Package foo is currently active from Git repository "..${separator}foo.git". +Resolving dependencies... +* foo 2.0.0 (was 1.0.0 from git ..${separator}foo.git at ..... +Building package executables... +Built foo:foo. +Activated foo 2.0.0. +''';})); + } + + ) + )); // Should now run the hosted one. var pub = await pubRun(global: true, args: ['foo']); From 206a5a70a15052a844c7057726fa09c176e7190f Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 27 Sep 2022 14:09:23 +0000 Subject: [PATCH 32/51] Fix test --- .../activate_hosted_after_git_test.dart | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/test/global/activate/activate_hosted_after_git_test.dart b/test/global/activate/activate_hosted_after_git_test.dart index e3c5a455c..15819ca28 100644 --- a/test/global/activate/activate_hosted_after_git_test.dart +++ b/test/global/activate/activate_hosted_after_git_test.dart @@ -22,20 +22,14 @@ void main() { await runPub(args: ['global', 'activate', '-sgit', '../foo.git']); - await runPub(args: ['global', 'activate', 'foo'], output: - predicate((output) { - return (output as String).replaceFirst(RegExp(' [a-f0-9]{5}'), ' ....') == ''' -Package foo is currently active from Git repository "..${separator}foo.git". -Resolving dependencies... -* foo 2.0.0 (was 1.0.0 from git ..${separator}foo.git at ..... -Building package executables... -Built foo:foo. -Activated foo 2.0.0. -''';})); - } - - ) - )); + await runPub( + args: ['global', 'activate', 'foo'], + output: allOf([ + contains( + 'Package foo is currently active from Git repository "..${separator}foo.git".'), + contains('* foo 2.0.0 (was 1.0.0 from git ..${separator}foo.git at'), + contains('Activated foo 2.0.0.') + ])); // Should now run the hosted one. var pub = await pubRun(global: true, args: ['foo']); From 15013b757946b2b252e4b747c005152f9ee3eb83 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Fri, 30 Sep 2022 07:38:17 +0000 Subject: [PATCH 33/51] Adjust messages, move check, fixed time comparison --- lib/src/command/cache_add.dart | 2 +- lib/src/command/dependency_services.dart | 12 +--- lib/src/entrypoint.dart | 18 ++---- lib/src/global_packages.dart | 6 +- lib/src/null_safety_analysis.dart | 6 +- lib/src/solver/report.dart | 46 +++++++++++++-- lib/src/solver/result.dart | 75 ++++++------------------ lib/src/source/cached.dart | 6 +- lib/src/source/git.dart | 11 ++-- lib/src/source/hosted.dart | 47 ++++++++++----- lib/src/system_cache.dart | 11 ++-- lib/src/utils.dart | 14 ++++- test/content_hash_test.dart | 4 +- 13 files changed, 126 insertions(+), 132 deletions(-) diff --git a/lib/src/command/cache_add.dart b/lib/src/command/cache_add.dart index 3c79d1d5c..ea8a8c162 100644 --- a/lib/src/command/cache_add.dart +++ b/lib/src/command/cache_add.dart @@ -77,7 +77,7 @@ class CacheAddCommand extends PubCommand { } // Download it. - await cache.downloadPackage(id, allowOutdatedHashChecks: true); + await cache.downloadPackage(id); } if (argResults['all']) { diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index d49c3dbce..6a40d37de 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -484,17 +484,7 @@ class DependencyServicesApplyCommand extends PubCommand { // This happens when we resolved a package from a legacy server // not providing archive_sha256. As a side-effect of downloading // the package we compute and store the sha256. - await cache.downloadPackage( - package, - allowOutdatedHashChecks: false, - ); - listedId = PackageId( - listedId.name, - listedId.version, - description.withSha256( - cache.hosted.sha256FromCache(listedId, cache), - ), - ); + listedId = await cache.downloadPackage(package); } updatedPackages.add(listedId); } else { diff --git a/lib/src/entrypoint.dart b/lib/src/entrypoint.dart index 9f1e4bf49..b0f007745 100644 --- a/lib/src/entrypoint.dart +++ b/lib/src/entrypoint.dart @@ -298,10 +298,6 @@ class Entrypoint { /// shown --- in case of failure, a reproduction command is shown. /// /// Updates [lockFile] and [packageRoot] accordingly. - /// - /// If [enforceLockfile] is true no changes to the current lockfile are - /// allowed. Instead the existing lockfile is loaded, verified against - /// pubspec.yaml and all dependencies downloaded. Future acquireDependencies( SolveType type, { Iterable? unlock, @@ -360,17 +356,11 @@ class Entrypoint { // We have to download files also with --dry-run to ensure we know the // archive hashes for downloaded files. - final newLockFile = await result.downloadCachedPackages(cache, - allowOutdatedHashChecks: true); + final newLockFile = await result.downloadCachedPackages(cache); final report = SolveReport( - type, - root, - lockFile, - newLockFile, - result.availableVersions, - cache, - ); + type, root, lockFile, newLockFile, result.availableVersions, cache, + dryRun: dryRun); if (!onlyReportSuccessOrFailure) { await report.show(); } @@ -383,7 +373,7 @@ class Entrypoint { if (onlyReportSuccessOrFailure) { log.message('Got dependencies$suffix.'); } else { - await report.summarize(dryRun: dryRun); + await report.summarize(); } if (!dryRun) { diff --git a/lib/src/global_packages.dart b/lib/src/global_packages.dart index fbec3c17d..038fe2337 100644 --- a/lib/src/global_packages.dart +++ b/lib/src/global_packages.dart @@ -224,10 +224,7 @@ class GlobalPackages { // We want the entrypoint to be rooted at 'dep' not the dummy-package. result.packages.removeWhere((id) => id.name == 'pub global activate'); - final lockFile = await result.downloadCachedPackages( - cache, - allowOutdatedHashChecks: true, - ); + final lockFile = await result.downloadCachedPackages(cache); final sameVersions = originalLockFile != null && originalLockFile.samePackageIds(lockFile); @@ -247,6 +244,7 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam lockFile, result.availableVersions, cache, + dryRun: false, ).show(); } diff --git a/lib/src/null_safety_analysis.dart b/lib/src/null_safety_analysis.dart index 450098450..fe89b559d 100644 --- a/lib/src/null_safety_analysis.dart +++ b/lib/src/null_safety_analysis.dart @@ -171,11 +171,7 @@ class NullSafetyAnalysis { if (source is CachedSource) { // TODO(sigurdm): Consider using withDependencyType here. - await source.downloadToSystemCache( - dependencyId, - _systemCache, - allowOutdatedHashChecks: true, - ); + await source.downloadToSystemCache(dependencyId, _systemCache); } final libDir = diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 4307bd0b8..258b3f2df 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -10,6 +10,7 @@ import '../lock_file.dart'; import '../log.dart' as log; import '../package.dart'; import '../package_name.dart'; +import '../source/hosted.dart'; import '../source/root.dart'; import '../system_cache.dart'; import '../utils.dart'; @@ -27,6 +28,7 @@ class SolveReport { final LockFile _previousLockFile; final LockFile _newLockFile; final SystemCache _cache; + final bool _dryRun; /// The available versions of all selected packages from their source. /// @@ -44,14 +46,50 @@ class SolveReport { this._previousLockFile, this._newLockFile, this._availableVersions, - this._cache, - ); + this._cache, { + required bool dryRun, + }) : _dryRun = dryRun; /// Displays a report of the results of the version resolution in /// [_newLockFile] relative to the [_previousLockFile] file. Future show() async { await _reportChanges(); await _reportOverrides(); + _checkContentHashesMatchOldLockfile(); + } + + _checkContentHashesMatchOldLockfile() { + for (final newId in _newLockFile.packages.values) { + var newDescription = newId.description; + // Use the cached content-hashes after downloading to ensure that + // content-hashes from legacy servers gets used. + if (newDescription is ResolvedHostedDescription) { + final cachedHash = newDescription.sha256; + assert(cachedHash != null); + final oldId = _previousLockFile.packages[newId.name]; + if (oldId != null && + cachedHash != null && + oldId.version == newId.version) { + final oldDecription = oldId.description; + if (oldDecription is ResolvedHostedDescription && + oldDecription.description == newDescription.description) { + final oldHash = oldDecription.sha256; + if (oldHash != null && !fixedTimeBytesEquals(cachedHash, oldHash)) { + log.warning(''' +The content of ${newId.name}-${newId.version} from ${newDescription.description.url} doesn't match the pubspec.lock. + +This indicates one of: +* The content has changed on the server since you created the pubspec.lock. +* The pubspec.lock has been corrupted. +${_dryRun ? '' : '\nThe pubspec.lock has been updated.'} + +See $contentHashesDocumentationUrl for more information. +'''); + } + } + } + } + } } /// Displays a one-line message summarizing what changes were made (or would @@ -64,7 +102,7 @@ class SolveReport { /// If [type] is `SolveType.UPGRADE` it also shows the number of packages that /// are not at the latest available version and the number of outdated /// packages. - Future summarize({bool dryRun = false}) async { + Future summarize() async { // Count how many dependencies actually changed. var dependencies = _newLockFile.packages.keys.toSet(); dependencies.addAll(_previousLockFile.packages.keys); @@ -90,7 +128,7 @@ class SolveReport { } } - if (dryRun) { + if (_dryRun) { if (numChanged == 0) { log.message('No dependencies would change$suffix.'); } else if (numChanged == 1) { diff --git a/lib/src/solver/result.dart b/lib/src/solver/result.dart index 2ac3939ab..1ba97d093 100644 --- a/lib/src/solver/result.dart +++ b/lib/src/solver/result.dart @@ -2,8 +2,6 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:typed_data'; - import 'package:collection/collection.dart'; import 'package:pub_semver/pub_semver.dart'; @@ -18,7 +16,6 @@ import '../pubspec.dart'; import '../source/cached.dart'; import '../source/hosted.dart'; import '../system_cache.dart'; -import '../utils.dart'; /// The result of a successful version resolution. class SolveResult { @@ -64,20 +61,23 @@ class SolveResult { /// If there is a mismatch between the previous content-hash from pubspec.lock /// and the new one a warning will be printed but the new one will be /// returned. - Future downloadCachedPackages( - SystemCache cache, { - required bool allowOutdatedHashChecks, - }) async { - await Future.wait(packages.map((id) async { - if (id.source is CachedSource) { - await withDependencyType(_root.dependencyType(id.name), () async { - await cache.downloadPackage( - id, - allowOutdatedHashChecks: allowOutdatedHashChecks, - ); - }); - } - })); + Future downloadCachedPackages(SystemCache cache) async { + final resolvedPackageIds = await Future.wait( + packages.map((id) async { + if (id.source is CachedSource) { + return await withDependencyType(_root.dependencyType(id.name), + () async { + return await cache.downloadPackage( + id, + ); + }); + } + return id; + }), + ); + + // Invariant: the content-hashes in PUB_CACHE matches those provided by the + // server. // Don't factor in overridden dependencies' SDK constraints, because we'll // accept those packages even if their constraints don't match. @@ -94,46 +94,7 @@ class SolveResult { }); } return LockFile( - packages.map((id) { - var description = id.description; - // Use the cached content-hashes after downloading to ensure that - // content-hashes from legacy servers gets used. - if (description is ResolvedHostedDescription) { - Uint8List? cachedHash = - description.description.source.sha256FromCache(id, cache); - if (cachedHash == null) { - // This should not happen. - throw StateError( - 'Archive for ${id.name}-${id.version} has no content hash.'); - } - final originalEntry = _previousLockFile.packages[id.name]; - if (originalEntry != null && originalEntry.version == id.version) { - final originalDescription = originalEntry.description; - if (originalDescription is ResolvedHostedDescription) { - final Uint8List? originalHash = originalDescription.sha256; - if (originalHash != null) { - if (!bytesEquals(cachedHash, originalHash)) { - log.warning(''' -The content of ${id.name}-${id.version} on the server doesn't match what was locked in your pubspec.lock. - -This might indicate that: -* The content has changed on the server since you created the pubspec.lock. -* The pubspec.lock has been corrupted. - -Your pubspec.lock has been updated according to what is on the server. - -See $contentHashesDocumentationUrl for more information. -'''); - } - } - } - } - - return PackageId( - id.name, id.version, description.withSha256(cachedHash)); - } - return id; - }).toList(), + resolvedPackageIds, sdkConstraints: sdkConstraints, mainDependencies: MapKeySet(_root.dependencies), devDependencies: MapKeySet(_root.devDependencies), diff --git a/lib/src/source/cached.dart b/lib/src/source/cached.dart index af85b829d..54630c73d 100644 --- a/lib/src/source/cached.dart +++ b/lib/src/source/cached.dart @@ -55,11 +55,7 @@ abstract class CachedSource extends Source { dirExists(getDirectoryInCache(id, cache)); /// Downloads the package identified by [id] to the system cache. - Future downloadToSystemCache( - PackageId id, - SystemCache cache, { - required bool allowOutdatedHashChecks, - }); + Future downloadToSystemCache(PackageId id, SystemCache cache); /// Returns the [Package]s that have been downloaded to the system cache. List getCachedPackages(SystemCache cache); diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 791c80a74..24b1294aa 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -298,11 +298,10 @@ class GitSource extends CachedSource { /// itself; each of the commit-specific directories are clones of a directory /// in `cache/`. @override - Future downloadToSystemCache( + Future downloadToSystemCache( PackageId id, - SystemCache cache, { - required bool allowOutdatedHashChecks, - }) async { + SystemCache cache, + ) async { return await _pool.withResource(() async { final ref = id.toRef(); final description = ref.description; @@ -322,8 +321,7 @@ class GitSource extends CachedSource { var revisionCachePath = _revisionCachePath(id, cache); final path = description.path; - return await _revisionCacheClones.putIfAbsent(revisionCachePath, - () async { + await _revisionCacheClones.putIfAbsent(revisionCachePath, () async { if (!entryExists(revisionCachePath)) { await _clone(_repoCachePath(ref, cache), revisionCachePath); await _checkOut(revisionCachePath, resolvedRef); @@ -332,6 +330,7 @@ class GitSource extends CachedSource { _updatePackageList(revisionCachePath, path); } }); + return id; }); } diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index d248ce70e..bdd605b23 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -698,11 +698,8 @@ class HostedSource extends CachedSource { /// `pub get` with a filled cache to be a fast case that doesn't require any /// new version-listings. @override - Future downloadToSystemCache( - PackageId id, - SystemCache cache, { - required bool allowOutdatedHashChecks, - }) async { + Future downloadToSystemCache( + PackageId id, SystemCache cache) async { final packageDir = getDirectoryInCache(id, cache); // Use the content-hash from the version-info to compare with what we @@ -714,13 +711,18 @@ class HostedSource extends CachedSource { // We allow the version-listing to be a few days outdated in order for `pub // get` with an existing working resolution and everything in cache to be // fast. - final versionInfo = await _versionInfo(id.toRef(), id.version, cache, - maxAge: allowOutdatedHashChecks ? Duration(days: 3) : null); + final versionInfo = await _versionInfo( + id.toRef(), + id.version, + cache, + maxAge: Duration(days: 3), + ); final expectedContentHash = versionInfo?.archiveSha256 ?? // Handling of legacy server - we use the hash from the id (typically // from the lockfile) to compare to the existing download. (id.description as ResolvedHostedDescription).sha256; + Uint8List? contentHash; if (!fileExists(hashPath(id, cache))) { if (dirExists(packageDir) && !cache.isOffline) { log.fine( @@ -728,24 +730,36 @@ class HostedSource extends CachedSource { deleteEntry(packageDir); } } else if (expectedContentHash == null) { + // Can happen with a legacy server combined with a legacy lock file. log.fine( 'Content-hash of ${id.name}-${id.version} not known from resolution.'); } else { - if (!bytesEquals(sha256FromCache(id, cache), expectedContentHash)) { + final hashFromCache = sha256FromCache(id, cache); + if (!fixedTimeBytesEquals(hashFromCache, expectedContentHash)) { log.warning( 'Cached version of ${id.name}-${id.version} has wrong hash - redownloading.'); if (cache.isOffline) { - fail('Cannot redownload while offline - try again online.'); + fail('Cannot redownload while offline. Try again without --offline.'); } deleteEntry(packageDir); + } else { + contentHash = hashFromCache; } } - if (!dirExists(packageDir)) { + if (dirExists(packageDir)) { + contentHash ??= sha256FromCache(id, cache); + } else { if (cache.isOffline) { - throw StateError('Cannot download packages when offline.'); + fail( + 'Missing package ${id.name}-${id.version}. Try again without --offline.'); } - await _download(id, packageDir, cache); + contentHash = await _download(id, packageDir, cache); } + return PackageId( + id.name, + id.version, + (id.description as ResolvedHostedDescription).withSha256(contentHash), + ); } /// Determines if the package identified by [id] is already downloaded to the @@ -953,7 +967,9 @@ class HostedSource extends CachedSource { /// If there is no archive_url, try to fetch it from /// `$server/packages/$package/versions/$version.tar.gz` where server comes /// from `id.description`. - Future _download( + /// + /// Returns the content-hash of the downloaded archive. + Future _download( PackageId id, String destPath, SystemCache cache, @@ -976,6 +992,7 @@ class HostedSource extends CachedSource { versions.firstWhereOrNull((i) => i.version == id.version); final packageName = id.name; final version = id.version; + late Uint8List contentHash; if (versionInfo == null) { throw PackageNotFoundException( 'Package $packageName has no version $version'); @@ -986,7 +1003,7 @@ class HostedSource extends CachedSource { log.fine('Downloading ${log.bold(id.name)} ${id.version}...'); // Download and extract the archive to a temp directory. - await withTempDir((tempDirForArchive) async { + return await withTempDir((tempDirForArchive) async { var archivePath = p.join(tempDirForArchive, '$packageName-$version.tar.gz'); var response = await withAuthenticatedClient( @@ -1023,6 +1040,7 @@ See $contentHashesDocumentationUrl. path, hexEncode(actualHash.bytes), ); + contentHash = Uint8List.fromList(actualHash.bytes); } // We download the archive to disk instead of streaming it directly into @@ -1056,6 +1074,7 @@ See $contentHashesDocumentationUrl. // another pub process has installed the same package version while we // downloaded. tryRenameDir(tempDir, destPath); + return contentHash; }); } diff --git a/lib/src/system_cache.dart b/lib/src/system_cache.dart index 4b3ee43da..11a10a174 100644 --- a/lib/src/system_cache.dart +++ b/lib/src/system_cache.dart @@ -221,16 +221,15 @@ class SystemCache { /// response if present instead of probing the server. Not probing allows for /// `pub get` with a filled cache to be a fast case that doesn't require any /// new version-listings. - Future downloadPackage( - PackageId id, { - required bool allowOutdatedHashChecks, - }) async { + /// + /// Returns [id] with an updated [ResolvedDescription], this can be different + /// if the content-hash changed while downloading. + Future downloadPackage(PackageId id) async { final source = id.source; assert(source is CachedSource); - await (source as CachedSource).downloadToSystemCache( + return await (source as CachedSource).downloadToSystemCache( id, this, - allowOutdatedHashChecks: allowOutdatedHashChecks, ); } diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 81ef9cfbf..12309ecce 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -9,7 +9,6 @@ import 'dart:io'; import 'dart:math' as math; import 'dart:typed_data'; -import 'package:collection/collection.dart'; import 'package:convert/convert.dart'; import 'package:crypto/crypto.dart' as crypto; import 'package:pub_semver/pub_semver.dart'; @@ -646,6 +645,15 @@ Map mapMap( }; } -bool bytesEquals(List? a, List? b) { - return const ListEquality().equals(a, b); +/// Compares two lists. If the lists have equal length this comparison will +/// iterate all elements, thus taking a fixed amount of time making timing +/// attacks harder. +bool fixedTimeBytesEquals(List? a, List? b) { + if (a == null || b == null) return a == b; + if (a.length != b.length) return false; + bool e = true; + for (var i = 0; i < a.length; i++) { + e &= a[i] == b[i]; + } + return e; } diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 5c2d5f7c0..186d12584 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -79,7 +79,7 @@ Future main() async { warning: allOf( contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( - 'Content of foo-1.0.0 has changed compared to what was locked your pubspec.lock.'), + 'The content of foo-1.0.0 from ${globalServer.url} doesn\'t match the pubspec.lock.'), ), exitCode: exit_codes.SUCCESS, ); @@ -106,7 +106,7 @@ Future main() async { await pubGet( warning: contains( - 'Content of foo-1.0.0 has changed compared to what was locked your pubspec.lock.', + 'The content of foo-1.0.0 from ${globalServer.url} doesn\'t match the pubspec.lock.', ), exitCode: exit_codes.SUCCESS, ); From f573b6189391d980b8cb52aedc8f51a82da5f508 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 15:42:49 +0200 Subject: [PATCH 34/51] Update lib/src/utils.dart Co-authored-by: Jonas Finnemann Jensen --- lib/src/utils.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index afcd509b2..0f1bf5172 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -651,11 +651,11 @@ Map mapMap( bool fixedTimeBytesEquals(List? a, List? b) { if (a == null || b == null) return a == b; if (a.length != b.length) return false; - bool e = true; + var e = 0; for (var i = 0; i < a.length; i++) { - e &= a[i] == b[i]; + e |= a[i] ^ b[i]; } - return e; + return e == 0; } /// Call [fn] retrying so long as [retryIf] return `true` for the exception From aedd523f77ff5292b2c632b9ecfceeff8863f92f Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 15:43:28 +0200 Subject: [PATCH 35/51] Update lib/src/source/hosted.dart Co-authored-by: Jonas Finnemann Jensen --- lib/src/source/hosted.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index dd80e5f04..1f9323b72 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -337,7 +337,7 @@ class HostedSource extends CachedSource { var pubspec = Pubspec.fromMap(pubspecData, cache.sources, expectedName: ref.name, location: location); final archiveSha256 = map['archive_sha256']; - if (archiveSha256 != null && archiveSha256 is! String) { + if (archiveSha256 is! String) { throw FormatException('archive_sha256 must be a String'); } final archiveUrl = map['archive_url']; From ae331d7713d68df5f974bd9c566b7fdcd391284e Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 15:45:51 +0200 Subject: [PATCH 36/51] Show all content mismatches Co-authored-by: Jonas Finnemann Jensen --- lib/src/solver/report.dart | 73 +++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 258b3f2df..9b5607bef 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -58,37 +58,60 @@ class SolveReport { _checkContentHashesMatchOldLockfile(); } - _checkContentHashesMatchOldLockfile() { - for (final newId in _newLockFile.packages.values) { - var newDescription = newId.description; +_checkContentHashesMatchOldLockfile() { + final issues = []; + + final newPackageNames = _newLockFile.packages.keys.toSet(); + final oldPackageNames = _previousLockFile.packages.keys.toSet(); + // We only care about packages that exist in both new and old lockfile. + for (final name in newPackageNames.intersection(oldPackageNames)) { + final newId = _newLockFile.packages[name]!; + final oldId = _previousLockFile.packages[name]!; + + // We only care about hosted packages + final newDescription = newId.description; + final oldDescription = oldId.description; + if (newDescription is! ResolvedHostedDescription || + oldDescription is! ResolvedHostedDescription) { + continue; + } + + // We don't care about changes in the hash if the version number changed! + if (newId.version != oldId.version) { + continue; + } + // Use the cached content-hashes after downloading to ensure that // content-hashes from legacy servers gets used. - if (newDescription is ResolvedHostedDescription) { - final cachedHash = newDescription.sha256; - assert(cachedHash != null); - final oldId = _previousLockFile.packages[newId.name]; - if (oldId != null && - cachedHash != null && - oldId.version == newId.version) { - final oldDecription = oldId.description; - if (oldDecription is ResolvedHostedDescription && - oldDecription.description == newDescription.description) { - final oldHash = oldDecription.sha256; - if (oldHash != null && !fixedTimeBytesEquals(cachedHash, oldHash)) { - log.warning(''' -The content of ${newId.name}-${newId.version} from ${newDescription.description.url} doesn't match the pubspec.lock. + final cachedHash = newDescription.sha256; + assert(cachedHash != null); + + // Ignore cases where the old lockfile doesn't have a content-hash + final oldHash = oldDescription.sha256; + if (oldHash == null) { + continue; + } + + if (!fixedTimeBytesEquals(cachedHash, oldHash)) { + issues.add( + '$name-${newId.version} from "${newDescription.description.url}"', + ); + } + } + + if (issues.isNotEmpty) { + log.warning(''' +The existing content-hash from pubspec.lock doesn't match contents for: + * ${issues.join('\n * ')} This indicates one of: -* The content has changed on the server since you created the pubspec.lock. -* The pubspec.lock has been corrupted. -${_dryRun ? '' : '\nThe pubspec.lock has been updated.'} + * The content has changed on the server since you created the pubspec.lock. + * The pubspec.lock has been corrupted. +${_dryRun ? '' : '\nThe content-hashes in pubspec.lock has been updated.'} -See $contentHashesDocumentationUrl for more information. +For more information see: +$contentHashesDocumentationUrl '''); - } - } - } - } } } From 79c75be0242d1e11236d0a0a155b91e2e83a5da7 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 15:47:24 +0200 Subject: [PATCH 37/51] Update test/content_hash_test.dart Co-authored-by: Jonas Finnemann Jensen --- test/content_hash_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 186d12584..832f86e2c 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -161,6 +161,7 @@ Future main() async { file('foo-1.0.0.sha256', originalHash), ]).validate(); }); + test( 'Legacy lockfile without content-hashes is updated with the hash on pub get on legacy server without content-hashes', () async { From 2b091aaaabd39121161f6ab065ccce5431425bdd Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 15:50:02 +0200 Subject: [PATCH 38/51] Update lib/src/command/dependency_services.dart Co-authored-by: Jonas Finnemann Jensen --- lib/src/command/dependency_services.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index 6a40d37de..9a49078e3 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -478,7 +478,7 @@ class DependencyServicesApplyCommand extends PubCommand { // but we don't want to download all archives - so we copy it from // the version listing. var listedId = (await cache.getVersions(package.toRef())) - .firstWhere((id) => id == package); + .firstWhere((id) => id == package, orElse: () => package); if ((listedId.description as ResolvedHostedDescription).sha256 == null) { // This happens when we resolved a package from a legacy server From b1de661664853f3fcc2175f63ab7037a1d8c5ae4 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 13:58:02 +0000 Subject: [PATCH 39/51] Use fixedTimeBytesEquals one more place --- lib/src/source/hosted.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 1f9323b72..d558d5bfc 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -1264,7 +1264,7 @@ class ResolvedHostedDescription extends ResolvedDescription { // Therefore we have to assume it is equal to any known value. (sha256 == null || other.sha256 == null || - const ListEquality().equals(sha256, other.sha256)); + fixedTimeBytesEquals(sha256, other.sha256)); } ResolvedHostedDescription withSha256(Uint8List? newSha256) => From 142222dbb575e879e6a4c267f16014d51b6693c1 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 14:10:38 +0000 Subject: [PATCH 40/51] Retry validation errors --- lib/src/source/hosted.dart | 2 +- test/content_hash_test.dart | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index d558d5bfc..48e9b0e21 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -1025,7 +1025,7 @@ class HostedSource extends CachedSource { if (expectedHash != null && output.value != expectedHash) { log.fine( 'Expected content-hash for ${id.name}-${id.version} $expectedHash actual: ${output.value}.'); - throw FormatException(''' + throw PackageIntegrityException(''' Downloaded archive for ${id.name}-${id.version} had wrong content-hash. This indicates a problem on the package repository: `${description.url}`. diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 832f86e2c..8a408c953 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -54,9 +54,12 @@ Future main() async { 'e7a7a0f6d9873e4c40cf68cc3cc9ca5b6c8cef6a2220241bdada4b9cb0083279'); await appDir({'foo': 'any'}).create(); await pubGet( + silent: contains('Retry #2'), error: contains('Downloaded archive for foo-1.0.0 had wrong content-hash.'), - exitCode: exit_codes.DATA, + environment: { + 'PUB_MAX_HTTP_RETRIES': '2', + }, ); }); From 8f73023ba4dc4588b98684e0957ea6eaef8ea888 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 14:29:03 +0000 Subject: [PATCH 41/51] Revert "Update lib/src/source/hosted.dart" This reverts commit aedd523f77ff5292b2c632b9ecfceeff8863f92f. --- lib/src/source/hosted.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 48e9b0e21..bb617d52f 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -337,7 +337,7 @@ class HostedSource extends CachedSource { var pubspec = Pubspec.fromMap(pubspecData, cache.sources, expectedName: ref.name, location: location); final archiveSha256 = map['archive_sha256']; - if (archiveSha256 is! String) { + if (archiveSha256 != null && archiveSha256 is! String) { throw FormatException('archive_sha256 must be a String'); } final archiveUrl = map['archive_url']; From 81902430d1f6eb9a8b2985cc95b9e2159fd39e8e Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 14:40:08 +0000 Subject: [PATCH 42/51] Adjust expectations --- test/content_hash_test.dart | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/content_hash_test.dart b/test/content_hash_test.dart index 8a408c953..6a0725bf7 100644 --- a/test/content_hash_test.dart +++ b/test/content_hash_test.dart @@ -82,7 +82,8 @@ Future main() async { warning: allOf( contains('Cached version of foo-1.0.0 has wrong hash - redownloading.'), contains( - 'The content of foo-1.0.0 from ${globalServer.url} doesn\'t match the pubspec.lock.'), + 'The existing content-hash from pubspec.lock doesn\'t match contents for:'), + contains('* foo-1.0.0 from "${server.url}"\n'), ), exitCode: exit_codes.SUCCESS, ); @@ -108,9 +109,12 @@ Future main() async { .deleteSync(); await pubGet( - warning: contains( - 'The content of foo-1.0.0 from ${globalServer.url} doesn\'t match the pubspec.lock.', - ), + warning: allOf([ + contains( + 'The existing content-hash from pubspec.lock doesn\'t match contents for:', + ), + contains('* foo-1.0.0 from "${globalServer.url}"'), + ]), exitCode: exit_codes.SUCCESS, ); final lockfile = loadYaml( From e93db2a027c87aaba3338ad7204c5919ed8a2405 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Thu, 6 Oct 2022 14:54:52 +0000 Subject: [PATCH 43/51] WIP --- lib/src/command/dependency_services.dart | 62 +++++++++++++++++------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index 9a49078e3..df75f67d9 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -358,6 +358,7 @@ class DependencyServicesApplyCommand extends PubCommand { : null; final lockFileYaml = lockFile == null ? null : loadYaml(lockFile); final lockFileEditor = lockFile == null ? null : YamlEditor(lockFile); + final hasContentHashes = _lockFileHasContentHashes(lockFileYaml); for (final p in toApply) { final targetPackage = p.name; final targetVersion = p.version; @@ -468,23 +469,30 @@ class DependencyServicesApplyCommand extends PubCommand { for (final package in solveResult.packages) { if (package.isRoot) continue; final description = package.description; - if (description is ResolvedHostedDescription && - description.sha256 == null) { - // We removed the hash above before resolution - as we get the - // locked id back we need to find the content-hash from the - // version listing. - // - // `pub get` gets this version-listing from the downloaded archive - // but we don't want to download all archives - so we copy it from - // the version listing. - var listedId = (await cache.getVersions(package.toRef())) - .firstWhere((id) => id == package, orElse: () => package); - if ((listedId.description as ResolvedHostedDescription).sha256 == - null) { - // This happens when we resolved a package from a legacy server - // not providing archive_sha256. As a side-effect of downloading - // the package we compute and store the sha256. - listedId = await cache.downloadPackage(package); + + if (description is ResolvedHostedDescription) { + if (hasContentHashes) { + if (description.sha256 == null) { + // We removed the hash above before resolution - as we get the + // locked id back we need to find the content-hash from the + // version listing. + // + // `pub get` gets this version-listing from the downloaded archive + // but we don't want to download all archives - so we copy it from + // the version listing. + var listedId = (await cache.getVersions(package.toRef())) + .firstWhere((id) => id == package, orElse: () => package); + if ((listedId.description as ResolvedHostedDescription) + .sha256 == + null) { + // This happens when we resolved a package from a legacy server + // not providing archive_sha256. As a side-effect of downloading + // the package we compute and store the sha256. + listedId = await cache.downloadPackage(package); + } + } + } else { + listedId = } updatedPackages.add(listedId); } else { @@ -583,3 +591,23 @@ VersionConstraint _compatibleWithIfPossible(VersionRange versionRange) { } return versionRange; } + +/// `true` iff any of the packages described by the [lockfile] has a +/// content-hash. +/// +/// Undefined for invalid lock files, but mostly `true`. +bool _lockFileHasContentHashes(dynamic lockfile) { + if (lockfile is! Map) return true; + final packages = lockfile['packages']; + if (packages is! Map) return true; + + /// We consider an empty lockfile ready to get content-hashes. + if (packages.isEmpty) return true; + for (final package in packages.values) { + if (package is! Map) return true; + final descriptor = package['description']; + if (descriptor is! Map) return true; + if (descriptor['sha256'] != null) return true; + } + return false; +} From 11451579a4a96285c039046a0beae33884df5a95 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:03:14 +0000 Subject: [PATCH 44/51] Don't add content-hashes in dependency-services unless they are already present --- lib/src/command/dependency_services.dart | 40 +++++++++++------- .../dependency_services_test.dart | 42 +++++++++++++++++++ .../Adding transitive.txt | 1 - .../dependency_services_test/Compatible.txt | 1 - .../Relative paths are allowed.txt | 1 - .../Removing transitive.txt | 1 - .../multibreaking.txt | 2 - 7 files changed, 68 insertions(+), 20 deletions(-) diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index df75f67d9..465218e51 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -398,9 +398,14 @@ class DependencyServicesApplyCommand extends PubCommand { ['packages', targetPackage, 'version'], targetVersion.toString()); // Remove the now outdated content-hash - it will be restored below // after resolution. - lockFileEditor.remove( - ['packages', targetPackage, 'description', 'sha256'], - ); + if (lockFileEditor + .parseAt(['packages', targetPackage, 'description']) + .value + .containsKey('sha256')) { + lockFileEditor.remove( + ['packages', targetPackage, 'description', 'sha256'], + ); + } } else if (targetRevision != null && lockFileYaml['packages'].containsKey(targetPackage)) { final ref = entrypoint.lockFile.packages[targetPackage]!.toRef(); @@ -466,38 +471,45 @@ class DependencyServicesApplyCommand extends PubCommand { // Only if we originally had a lock-file we write the resulting lockfile back. if (updatedLockfile != null) { final updatedPackages = []; - for (final package in solveResult.packages) { + for (var package in solveResult.packages) { if (package.isRoot) continue; final description = package.description; + // Handle content-hashes of hosted dependencies. if (description is ResolvedHostedDescription) { + // Ensure we get content-hashes if the original lock-file had + // them. if (hasContentHashes) { if (description.sha256 == null) { // We removed the hash above before resolution - as we get the // locked id back we need to find the content-hash from the // version listing. // - // `pub get` gets this version-listing from the downloaded archive - // but we don't want to download all archives - so we copy it from - // the version listing. + // `pub get` gets this version-listing from the downloaded + // archive but we don't want to download all archives - so we + // copy it from the version listing. var listedId = (await cache.getVersions(package.toRef())) .firstWhere((id) => id == package, orElse: () => package); if ((listedId.description as ResolvedHostedDescription) .sha256 == null) { - // This happens when we resolved a package from a legacy server - // not providing archive_sha256. As a side-effect of downloading - // the package we compute and store the sha256. + // This happens when we resolved a package from a legacy + // server not providing archive_sha256. As a side-effect of + // downloading the package we compute and store the sha256. listedId = await cache.downloadPackage(package); } } } else { - listedId = + // The original pubspec.lock did not have content-hashes. Remove + // any content hash, so we don't start adding them. + package = PackageId( + package.name, + package.version, + description.withSha256(null), + ); } - updatedPackages.add(listedId); - } else { - updatedPackages.add(package); } + updatedPackages.add(package); } final newLockFile = LockFile( diff --git a/test/dependency_services/dependency_services_test.dart b/test/dependency_services/dependency_services_test.dart index 5832a6644..312e023c9 100644 --- a/test/dependency_services/dependency_services_test.dart +++ b/test/dependency_services/dependency_services_test.dart @@ -10,8 +10,11 @@ import 'package:pub/src/io.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:test/test.dart'; +import 'package:yaml/yaml.dart'; +import 'package:yaml_edit/yaml_edit.dart'; import '../descriptor.dart' as d; +import '../descriptor.dart'; import '../golden_file.dart'; import '../test_pub.dart'; @@ -215,6 +218,45 @@ Future main() async { }); }); + testWithGolden('Preserves no content-hashes', (context) async { + final server = (await servePackages()) + ..serve('foo', '1.2.3') + ..serve('foo', '2.2.3') + ..serve('bar', '1.2.3') + ..serve('bar', '2.2.3') + ..serve('boo', '1.2.3') + ..serveContentHashes = true; + + await d.dir(appPath, [ + d.pubspec({ + 'name': 'app', + 'dependencies': { + 'foo': '^1.0.0', + 'bar': '^1.0.0', + 'boo': '^1.0.0', + }, + }) + ]).create(); + await pubGet(); + final lockFile = File(path(p.join(appPath, 'pubspec.lock'))); + final lockFileYaml = YamlEditor( + lockFile.readAsStringSync(), + ); + for (final p in lockFileYaml.parseAt(['packages']).value.entries) { + lockFileYaml.remove(['packages', p.key, 'description', 'sha256']); + } + lockFile.writeAsStringSync(lockFileYaml.toString()); + + server.serve('foo', '1.2.4'); + server.serve('boo', '1.2.4'); + + server.dontAllowDownloads(); + + await _listReportApply(context, [ + _PackageVersion('foo', '1.2.4'), + ]); + }); + testWithGolden('Adding transitive', (context) async { final server = (await servePackages()) ..serve('foo', '1.2.3') diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt index 3b16db157..831a5f96a 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt @@ -176,7 +176,6 @@ packages: dependency: "direct main" description: name: foo - sha256: fc06d01652f7b73f789abeb5b61aeb68b13cd472f87610cb8fb80e402a9139ff url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt index 4fbaffbd0..980aaf6c3 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt @@ -396,7 +396,6 @@ packages: dependency: "direct main" description: name: foo - sha256: "88f2f9251967bf04bd478873f074b9d8df9f1c959afc150ba3b0ea813d48161e" url: "http://localhost:$PORT" source: hosted version: "1.2.4" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt index 2e3877339..deef59ae6 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt @@ -182,7 +182,6 @@ packages: dependency: "direct main" description: name: foo - sha256: c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0 url: "http://localhost:$PORT" source: hosted version: "2.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt index 04986112b..3d8b2c739 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt @@ -214,7 +214,6 @@ packages: dependency: "direct main" description: name: foo - sha256: bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt index 3767075d5..93adb5ddc 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt @@ -378,7 +378,6 @@ packages: dependency: "direct main" description: name: bar - sha256: b8187621010649d6385788d7630adcd88d6548a7938899b6f18820961df3b879 url: "http://localhost:$PORT" source: hosted version: "2.0.0" @@ -394,7 +393,6 @@ packages: dependency: "direct main" description: name: foo - sha256: "2347a7792f73d0f8cc8aa41d4895317bd1745724b8bc77d8c03faf821c9059b7" url: "http://localhost:$PORT" source: hosted version: "3.0.1" From 7a38f4aa25243cd3810376de85e176651e688efb Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:09:43 +0000 Subject: [PATCH 45/51] Remove unused import --- test/dependency_services/dependency_services_test.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/test/dependency_services/dependency_services_test.dart b/test/dependency_services/dependency_services_test.dart index 312e023c9..02a32a098 100644 --- a/test/dependency_services/dependency_services_test.dart +++ b/test/dependency_services/dependency_services_test.dart @@ -10,7 +10,6 @@ import 'package:pub/src/io.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart' as shelf; import 'package:test/test.dart'; -import 'package:yaml/yaml.dart'; import 'package:yaml_edit/yaml_edit.dart'; import '../descriptor.dart' as d; From da649bf3d39b1c573a16d88e7c1900a4e949c8ff Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:18:21 +0000 Subject: [PATCH 46/51] Dartfmt --- lib/src/solver/report.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 9b5607bef..9feca3d18 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -58,7 +58,7 @@ class SolveReport { _checkContentHashesMatchOldLockfile(); } -_checkContentHashesMatchOldLockfile() { + _checkContentHashesMatchOldLockfile() { final issues = []; final newPackageNames = _newLockFile.packages.keys.toSet(); From dc34703acc21fda9142e54b8e9ffa1eed29aee1c Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:19:20 +0000 Subject: [PATCH 47/51] Missing return type --- lib/src/solver/report.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/solver/report.dart b/lib/src/solver/report.dart index 9feca3d18..176ab671d 100644 --- a/lib/src/solver/report.dart +++ b/lib/src/solver/report.dart @@ -58,7 +58,7 @@ class SolveReport { _checkContentHashesMatchOldLockfile(); } - _checkContentHashesMatchOldLockfile() { + void _checkContentHashesMatchOldLockfile() { final issues = []; final newPackageNames = _newLockFile.packages.keys.toSet(); From dd2e088654385e52fc9989ac1e4ac6b9189ec475 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:28:27 +0000 Subject: [PATCH 48/51] Add missing golden --- .../Preserves no content-hashes.txt | 384 ++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 test/testdata/goldens/dependency_services/dependency_services_test/Preserves no content-hashes.txt diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Preserves no content-hashes.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Preserves no content-hashes.txt new file mode 100644 index 000000000..bd707b488 --- /dev/null +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Preserves no content-hashes.txt @@ -0,0 +1,384 @@ +# GENERATED BY: test/dependency_services/dependency_services_test.dart + +$ cat pubspec.yaml +{"name":"app","dependencies":{"foo":"^1.0.0","bar":"^1.0.0","boo":"^1.0.0"},"environment":{"sdk":">=0.1.2 <1.0.0"}} +$ cat pubspec.lock +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + bar: + dependency: "direct main" + description: + name: bar + url: "http://localhost:$PORT" + source: hosted + version: "1.2.3" + boo: + dependency: "direct main" + description: + name: boo + url: "http://localhost:$PORT" + source: hosted + version: "1.2.3" + foo: + dependency: "direct main" + description: + name: foo + url: "http://localhost:$PORT" + source: hosted + version: "1.2.3" +sdks: + dart: ">=0.1.2 <1.0.0" +-------------------------------- END OF OUTPUT --------------------------------- + +## Section list +$ dependency_services list +{ + "dependencies": [ + { + "name": "bar", + "version": "1.2.3", + "kind": "direct", + "constraint": "^1.0.0", + "source": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT" + } + } + }, + { + "name": "boo", + "version": "1.2.3", + "kind": "direct", + "constraint": "^1.0.0", + "source": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT" + } + } + }, + { + "name": "foo", + "version": "1.2.3", + "kind": "direct", + "constraint": "^1.0.0", + "source": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT" + } + } + } + ] +} + +-------------------------------- END OF OUTPUT --------------------------------- + +## Section report +$ dependency_services report +{ + "dependencies": [ + { + "name": "bar", + "version": "1.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT" + } + }, + "latest": "2.2.3", + "constraint": "^1.0.0", + "compatible": [], + "singleBreaking": [ + { + "name": "bar", + "version": "2.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT", + "sha256": "adcfe9ac3d6955fd4332f29f47bf3e814e388e2da7c2bc55d4561971bf8b5335" + } + }, + "constraintBumped": "^2.2.3", + "constraintWidened": ">=1.0.0 <3.0.0", + "constraintBumpedIfNeeded": "^2.2.3", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT" + } + } + } + ], + "multiBreaking": [ + { + "name": "bar", + "version": "2.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT", + "sha256": "adcfe9ac3d6955fd4332f29f47bf3e814e388e2da7c2bc55d4561971bf8b5335" + } + }, + "constraintBumped": "^2.2.3", + "constraintWidened": ">=1.0.0 <3.0.0", + "constraintBumpedIfNeeded": "^2.2.3", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "bar", + "url": "http://localhost:$PORT" + } + } + } + ] + }, + { + "name": "boo", + "version": "1.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT" + } + }, + "latest": "1.2.4", + "constraint": "^1.0.0", + "compatible": [ + { + "name": "boo", + "version": "1.2.4", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT", + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" + } + }, + "constraintBumped": "^1.0.0", + "constraintWidened": "^1.0.0", + "constraintBumpedIfNeeded": "^1.0.0", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT" + } + } + } + ], + "singleBreaking": [ + { + "name": "boo", + "version": "1.2.4", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT", + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" + } + }, + "constraintBumped": "^1.2.4", + "constraintWidened": "^1.0.0", + "constraintBumpedIfNeeded": "^1.0.0", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT" + } + } + } + ], + "multiBreaking": [ + { + "name": "boo", + "version": "1.2.4", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT", + "sha256": "b060c0315b77c8383da5f9a7eee7667dbdc8108969e0a7855e294e35e7f42230" + } + }, + "constraintBumped": "^1.2.4", + "constraintWidened": "^1.0.0", + "constraintBumpedIfNeeded": "^1.0.0", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "boo", + "url": "http://localhost:$PORT" + } + } + } + ] + }, + { + "name": "foo", + "version": "1.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT" + } + }, + "latest": "2.2.3", + "constraint": "^1.0.0", + "compatible": [ + { + "name": "foo", + "version": "1.2.4", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT", + "sha256": "88f2f9251967bf04bd478873f074b9d8df9f1c959afc150ba3b0ea813d48161e" + } + }, + "constraintBumped": "^1.0.0", + "constraintWidened": "^1.0.0", + "constraintBumpedIfNeeded": "^1.0.0", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT" + } + } + } + ], + "singleBreaking": [ + { + "name": "foo", + "version": "2.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT", + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" + } + }, + "constraintBumped": "^2.2.3", + "constraintWidened": ">=1.0.0 <3.0.0", + "constraintBumpedIfNeeded": "^2.2.3", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT" + } + } + } + ], + "multiBreaking": [ + { + "name": "foo", + "version": "2.2.3", + "kind": "direct", + "source": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT", + "sha256": "bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e" + } + }, + "constraintBumped": "^2.2.3", + "constraintWidened": ">=1.0.0 <3.0.0", + "constraintBumpedIfNeeded": "^2.2.3", + "previousVersion": "1.2.3", + "previousConstraint": "^1.0.0", + "previousSource": { + "type": "hosted", + "description": { + "name": "foo", + "url": "http://localhost:$PORT" + } + } + } + ] + } + ] +} + +-------------------------------- END OF OUTPUT --------------------------------- + +## Section apply +$ echo '{"dependencyChanges":[{"name":"foo","version":"1.2.4"}]}' | dependency_services apply +{"dependencies":[]} + +-------------------------------- END OF OUTPUT --------------------------------- + +$ cat pubspec.yaml +{"name":"app","dependencies":{"foo":"^1.0.0","bar":"^1.0.0","boo":"^1.0.0"},"environment":{"sdk":">=0.1.2 <1.0.0"}} +$ cat pubspec.lock +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + bar: + dependency: "direct main" + description: + name: bar + url: "http://localhost:$PORT" + source: hosted + version: "1.2.3" + boo: + dependency: "direct main" + description: + name: boo + url: "http://localhost:$PORT" + source: hosted + version: "1.2.3" + foo: + dependency: "direct main" + description: + name: foo + url: "http://localhost:$PORT" + source: hosted + version: "1.2.4" +sdks: + dart: ">=0.1.2 <1.0.0" From a0567760b5c02facf1c485dc0a16f55576ea25b2 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:40:56 +0000 Subject: [PATCH 49/51] Fix in dependency_services --- lib/src/command/dependency_services.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/command/dependency_services.dart b/lib/src/command/dependency_services.dart index 465218e51..13c6e7013 100644 --- a/lib/src/command/dependency_services.dart +++ b/lib/src/command/dependency_services.dart @@ -488,15 +488,15 @@ class DependencyServicesApplyCommand extends PubCommand { // `pub get` gets this version-listing from the downloaded // archive but we don't want to download all archives - so we // copy it from the version listing. - var listedId = (await cache.getVersions(package.toRef())) + package = (await cache.getVersions(package.toRef())) .firstWhere((id) => id == package, orElse: () => package); - if ((listedId.description as ResolvedHostedDescription) + if ((package.description as ResolvedHostedDescription) .sha256 == null) { // This happens when we resolved a package from a legacy // server not providing archive_sha256. As a side-effect of // downloading the package we compute and store the sha256. - listedId = await cache.downloadPackage(package); + package = await cache.downloadPackage(package); } } } else { From 192d66eb896dbd81ba1bcde493fdea7e738f615a Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 10:54:12 +0000 Subject: [PATCH 50/51] Better import --- lib/src/source/hosted.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index bb617d52f..0def1edff 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -5,7 +5,6 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io' as io; -import 'dart:io'; import 'dart:math' as math; import 'dart:typed_data'; @@ -823,7 +822,7 @@ class HostedSource extends CachedSource { Uint8List? sha256FromCache(PackageId id, SystemCache cache) { try { return hexDecode(readTextFile(hashPath(id, cache))); - } on IOException { + } on io.IOException { return null; } } From 634695602ba15b338d53442a5df732021430a3c7 Mon Sep 17 00:00:00 2001 From: Sigurd Meldgaard Date: Tue, 18 Oct 2022 11:10:08 +0000 Subject: [PATCH 51/51] goldens --- .../dependency_services_test/Adding transitive.txt | 1 + .../dependency_services/dependency_services_test/Compatible.txt | 1 + .../dependency_services_test/Relative paths are allowed.txt | 1 + .../dependency_services_test/Removing transitive.txt | 1 + .../dependency_services_test/multibreaking.txt | 2 ++ 5 files changed, 6 insertions(+) diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt index 831a5f96a..3b16db157 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Adding transitive.txt @@ -176,6 +176,7 @@ packages: dependency: "direct main" description: name: foo + sha256: fc06d01652f7b73f789abeb5b61aeb68b13cd472f87610cb8fb80e402a9139ff url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt index 980aaf6c3..4fbaffbd0 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Compatible.txt @@ -396,6 +396,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "88f2f9251967bf04bd478873f074b9d8df9f1c959afc150ba3b0ea813d48161e" url: "http://localhost:$PORT" source: hosted version: "1.2.4" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt index deef59ae6..2e3877339 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Relative paths are allowed.txt @@ -182,6 +182,7 @@ packages: dependency: "direct main" description: name: foo + sha256: c3bda774737102f799574749076544dea1a4745b5c38d590d4f206f997bfe8a0 url: "http://localhost:$PORT" source: hosted version: "2.0.0" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt index 3d8b2c739..04986112b 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/Removing transitive.txt @@ -214,6 +214,7 @@ packages: dependency: "direct main" description: name: foo + sha256: bf378a3f6c4840f911d66ab375f6d3eae78a015a41f0b8b202c31d4af010892e url: "http://localhost:$PORT" source: hosted version: "2.2.3" diff --git a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt index 93adb5ddc..3767075d5 100644 --- a/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt +++ b/test/testdata/goldens/dependency_services/dependency_services_test/multibreaking.txt @@ -378,6 +378,7 @@ packages: dependency: "direct main" description: name: bar + sha256: b8187621010649d6385788d7630adcd88d6548a7938899b6f18820961df3b879 url: "http://localhost:$PORT" source: hosted version: "2.0.0" @@ -393,6 +394,7 @@ packages: dependency: "direct main" description: name: foo + sha256: "2347a7792f73d0f8cc8aa41d4895317bd1745724b8bc77d8c03faf821c9059b7" url: "http://localhost:$PORT" source: hosted version: "3.0.1"