Content hashing of archives

doc/repository-spec-v2.md

@@ -120,7 +120,7 @@ parse the `<message>`.
 The `dart pub` client allows users to save an opaque `<token>` for each
 `<hosted-url>`. When the `dart pub` client makes a request to a `<hosted-url>`
 for which it has a `<token>` stored, it will attach an `Authorization` header
-as follows: 
+as follows:
 
  * `Authorization: Bearer <token>`
 
@@ -229,6 +229,7 @@ server, this could work in many different ways.
     "version": "<version>",
     "retracted": true || false, /* optional field, false if omitted */
     "archive_url": "https://.../archive.tar.gz",
+    "archive_sha256": "95cbaad58e2cf32d1aa852f20af1fcda1820ead92a4b1447ea7ba1ba18195d27"
     "pubspec": {
       /* pubspec contents as JSON object */
     }
@@ -238,6 +239,7 @@ server, this could work in many different ways.
       "version": "<package>",
       "retracted": true || false, /* optional field, false if omitted */
       "archive_url": "https://.../archive.tar.gz",
+      "archive_sha256": "95cbaad58e2cf32d1aa852f20af1fcda1820ead92a4b1447ea7ba1ba18195d27"
       "pubspec": {
         /* pubspec contents as JSON object */
       }
@@ -256,6 +258,15 @@ parameters. This allows for the server to return signed-URLs for S3, GCS or
 other blob storage service. If temporary URLs are returned it is wise to not set
 expiration to less than 25 minutes (to allow for retries and clock drift).
 
+The `archive_sha256` should be the hex-encoded sha256 checksum of the file at
+archive_url. It is an optional field that allows the pub client to verify the
+integrity of downloaded archive.
+
+The `archive_sha256` also provides an easy way for clients to detect if
+something has changed on the server. In the absense of this field the client can
+still download the archive to obtain a checksum and detect changes to the
+archive.
+
 If `<hosted-url>` for the server returning `archive_url` is a prefix of
 `archive_url`, then the `Authorization: Bearer <token>` is also included when
 `archive_url` is requested. Example: if `https://pub.example.com/path` returns

lib/src/command/cache_add.dart

@@ -77,7 +77,7 @@ class CacheAddCommand extends PubCommand {
       }
 
       // Download it.
-      await cache.downloadPackage(id);
+      await cache.downloadPackage(id, allowOutdatedHashChecks: true);
     }
 
     if (argResults['all']) {

lib/src/command/dependency_services.dart

@@ -24,6 +24,7 @@ import '../pubspec.dart';
 import '../pubspec_utils.dart';
 import '../solver.dart';
 import '../source/git.dart';
+import '../source/hosted.dart';
 import '../system_cache.dart';
 import '../utils.dart';
 
@@ -394,6 +395,11 @@ class DependencyServicesApplyCommand extends PubCommand {
             lockFileYaml['packages'].containsKey(targetPackage)) {
           lockFileEditor.update(
               ['packages', targetPackage, 'version'], targetVersion.toString());
+          // Remove the now outdated content-hash - it will be restored below
+          // after resolution.
+          lockFileEditor.remove(
+            ['packages', targetPackage, 'description', 'sha256'],
+          );
         } else if (targetRevision != null &&
             lockFileYaml['packages'].containsKey(targetPackage)) {
           final ref = entrypoint.lockFile.packages[targetPackage]!.toRef();
@@ -457,8 +463,54 @@ class DependencyServicesApplyCommand extends PubCommand {
           writeTextFile(entrypoint.pubspecPath, updatedPubspec);
         }
         // Only if we originally had a lock-file we write the resulting lockfile back.
-        if (lockFileEditor != null) {
-          entrypoint.saveLockFile(solveResult);
+        if (updatedLockfile != null) {
+          final updatedPackages = <PackageId>[];
+          for (final package in solveResult.packages) {
+            if (package.isRoot) continue;
+            final description = package.description;
+            if (description is ResolvedHostedDescription &&
+                description.sha256 == null) {
+              // We removed the hash above before resolution - as we get the
+              // locked id back we need to find the content-hash from the
+              // version listing.
+              //
+              // `pub get` gets this version-listing from the downloaded archive
+              // but we don't want to download all archives - so we copy it from
+              // the version listing.
+              var listedId = (await cache.getVersions(package.toRef()))
+                  .firstWhere((id) => id == package);
+              if ((listedId.description as ResolvedHostedDescription).sha256 ==
+                  null) {
+                // This happens when we resolved a package from a legacy server
+                // not providing archive_sha256. As a side-effect of downloading
+                // the package we compute and store the sha256.
+                await cache.downloadPackage(
+                  package,
+                  allowOutdatedHashChecks: false,
+                );
+                listedId = PackageId(
+                  listedId.name,
+                  listedId.version,
+                  description.withSha256(
+                    cache.hosted.sha256FromCache(listedId, cache),
+                  ),
+                );
+              }
+              updatedPackages.add(listedId);
+            } else {
+              updatedPackages.add(package);
+            }
+          }
+
+          final newLockFile = LockFile(
+            updatedPackages,
+            sdkConstraints: updatedLockfile.sdkConstraints,
+            mainDependencies: pubspec.dependencies.keys.toSet(),
+            devDependencies: pubspec.devDependencies.keys.toSet(),
+            overriddenDependencies: pubspec.dependencyOverrides.keys.toSet(),
+          );
+
+          newLockFile.writeToFile(entrypoint.lockFilePath, cache);
         }
       },
     );

lib/src/command/get.dart

@@ -49,6 +49,7 @@ class GetCommand extends PubCommand {
       log.warning(log.yellow(
           'The --packages-dir flag is no longer used and does nothing.'));
     }
+
     await entrypoint.acquireDependencies(
       SolveType.get,
       dryRun: argResults['dry-run'],

lib/src/command/outdated.dart

@@ -204,7 +204,11 @@ class OutdatedCommand extends PubCommand {
         latestIsOverridden = true;
       }
 
-      final packageStatus = await current?.source.status(current, cache);
+      final packageStatus = await current?.source.status(
+        current.toRef(),
+        current.version,
+        cache,
+      );
       final discontinued =
           packageStatus == null ? false : packageStatus.isDiscontinued;
       final discontinuedReplacedBy = packageStatus?.discontinuedReplacedBy;

lib/src/entrypoint.dart

@@ -8,7 +8,6 @@ import 'dart:io';
 import 'dart:math';
 
 import 'package:collection/collection.dart';
-import 'package:meta/meta.dart';
 import 'package:path/path.dart' as p;
 import 'package:pool/pool.dart';
 import 'package:pub_semver/pub_semver.dart';
@@ -31,6 +30,7 @@ import 'pub_embeddable_command.dart';
 import 'pubspec.dart';
 import 'sdk.dart';
 import 'solver.dart';
+import 'solver/report.dart';
 import 'source/cached.dart';
 import 'source/unknown.dart';
 import 'system_cache.dart';
@@ -276,11 +276,11 @@ class Entrypoint {
   ///
   /// Performs version resolution according to [SolveType].
   ///
-  /// [useLatest], if provided, defines a list of packages that will be
-  /// unlocked and forced to their latest versions. If [upgradeAll] is
-  /// true, the previous lockfile is ignored and all packages are re-resolved
-  /// from scratch. Otherwise, it will attempt to preserve the versions of all
-  /// previously locked packages.
+  /// [useLatest], if provided, defines a list of packages that will be unlocked
+  /// and forced to their latest versions. If [upgradeAll] is true, the previous
+  /// lockfile is ignored and all packages are re-resolved from scratch.
+  /// Otherwise, it will attempt to preserve the versions of all previously
+  /// locked packages.
   ///
   /// Shows a report of the changes made relative to the previous lockfile. If
   /// this is an upgrade or downgrade, all transitive dependencies are shown in
@@ -290,10 +290,14 @@ class Entrypoint {
   /// If [precompile] is `true` (the default), this snapshots dependencies'
   /// executables.
   ///
-  /// if [onlyReportSuccessOrFailure] is `true` only success or failure will be shown ---
-  /// in case of failure, a reproduction command is shown.
+  /// if [onlyReportSuccessOrFailure] is `true` only success or failure will be
+  /// shown --- in case of failure, a reproduction command is shown.
   ///
   /// Updates [lockFile] and [packageRoot] accordingly.
+  ///
+  /// If [enforceLockfile] is true no changes to the current lockfile are
+  /// allowed. Instead the existing lockfile is loaded, verified against
+  /// pubspec.yaml and all dependencies downloaded.
   Future<void> acquireDependencies(
     SolveType type, {
     Iterable<String>? unlock,
@@ -350,17 +354,32 @@ class Entrypoint {
       }
     }
 
+    // We have to download files also with --dry-run to ensure we know the
+    // archive hashes for downloaded files.
+    final newLockFile = await result.downloadCachedPackages(cache,
+        allowOutdatedHashChecks: true);
+
+    final report = SolveReport(
+      type,
+      root,
+      lockFile,
+      newLockFile,
+      result.availableVersions,
+      cache,
+    );
     if (!onlyReportSuccessOrFailure) {
-      await result.showReport(type, cache);
+      await report.show();
     }
+    _lockFile = newLockFile;
+
     if (!dryRun) {
-      await result.downloadCachedPackages(cache);
-      saveLockFile(result);
+      newLockFile.writeToFile(lockFilePath, cache);
     }
+
     if (onlyReportSuccessOrFailure) {
       log.message('Got dependencies$suffix.');
     } else {
-      await result.summarizeChanges(type, cache, dryRun: dryRun);
+      await report.summarize(dryRun: dryRun);
     }
 
     if (!dryRun) {
@@ -820,21 +839,6 @@ class Entrypoint {
     }
   }
 
-  /// Saves a list of concrete package versions to the `pubspec.lock` file.
-  ///
-  /// Will use Windows line endings (`\r\n`) if a `pubspec.lock` exists, and
-  /// uses that.
-  void saveLockFile(SolveResult result) {
-    _lockFile = result.lockFile;
-
-    final windowsLineEndings = fileExists(lockFilePath) &&
-        detectWindowsLineEndings(readTextFile(lockFilePath));
-
-    final serialized = lockFile.serialize(root.dir);
-    writeTextFile(lockFilePath,
-        windowsLineEndings ? serialized.replaceAll('\n', '\r\n') : serialized);
-  }
-
   /// If the entrypoint uses the old-style `.pub` cache directory, migrates it
   /// to the new-style `.dart_tool/pub` directory.
   void migrateCache() {
@@ -913,22 +917,3 @@ See https://dart.dev/go/sdk-constraint
         '"pub" version, please run "$topLevelProgram pub get".');
   }
 }
-
-/// Returns `true` if the [text] looks like it uses windows line endings.
-///
-/// The heuristic used is to count all `\n` in the text and if stricly more than
-/// half of them are preceded by `\r` we report `true`.
-@visibleForTesting
-bool detectWindowsLineEndings(String text) {
-  var index = -1;
-  var unixNewlines = 0;
-  var windowsNewlines = 0;
-  while ((index = text.indexOf('\n', index + 1)) != -1) {
-    if (index != 0 && text[index - 1] == '\r') {
-      windowsNewlines++;
-    } else {
-      unixNewlines++;
-    }
-  }
-  return windowsNewlines > unixNewlines;
-}

lib/src/global_packages.dart

@@ -23,6 +23,7 @@ import 'sdk.dart';
 import 'sdk/dart.dart';
 import 'solver.dart';
 import 'solver/incompatibility_cause.dart';
+import 'solver/report.dart';
 import 'source/cached.dart';
 import 'source/git.dart';
 import 'source/hosted.dart';
@@ -178,7 +179,7 @@ class GlobalPackages {
     final tempDir = cache.createTempDir();
     // TODO(rnystrom): Look in "bin" and display list of binaries that
     // user can run.
-    _writeLockFile(tempDir, LockFile([id]));
+    LockFile([id]).writeToFile(p.join(tempDir, 'pubspec.lock'), cache);
 
     tryDeleteEntry(_packageDir(name));
     tryRenameDir(tempDir, _packageDir(name));
@@ -223,24 +224,34 @@ class GlobalPackages {
     // We want the entrypoint to be rooted at 'dep' not the dummy-package.
     result.packages.removeWhere((id) => id.name == 'pub global activate');
 
-    final sameVersions = originalLockFile != null &&
-        originalLockFile.samePackageIds(result.lockFile);
+    final lockFile = await result.downloadCachedPackages(
+      cache,
+      allowOutdatedHashChecks: true,
+    );
+    final sameVersions =
+        originalLockFile != null && originalLockFile.samePackageIds(lockFile);
 
-    final PackageId id = result.lockFile.packages[name]!;
+    final PackageId id = lockFile.packages[name]!;
     if (sameVersions) {
       log.message('''
 The package $name is already activated at newest available version.
 To recompile executables, first run `$topLevelProgram pub global deactivate $name`.
 ''');
     } else {
       // Only precompile binaries if we have a new resolution.
-      if (!silent) await result.showReport(SolveType.get, cache);
-
-      await result.downloadCachedPackages(cache);
+      if (!silent) {
+        await SolveReport(
+          SolveType.get,
+          root,
+          originalLockFile ?? LockFile.empty(),
+          lockFile,
+          result.availableVersions,
+          cache,
+        ).show();
+      }
 
-      final lockFile = result.lockFile;
       final tempDir = cache.createTempDir();
-      _writeLockFile(tempDir, lockFile);
+      lockFile.writeToFile(p.join(tempDir, 'pubspec.lock'), cache);
 
       // Load the package graph from [result] so we don't need to re-parse all
       // the pubspecs.
@@ -263,7 +274,7 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam
     final entrypoint = Entrypoint.global(
       _packageDir(id.name),
       cache.loadCached(id),
-      result.lockFile,
+      lockFile,
       cache,
       solveResult: result,
     );
@@ -276,11 +287,6 @@ To recompile executables, first run `$topLevelProgram pub global deactivate $nam
     if (!silent) log.message('Activated ${_formatPackage(id)}.');
   }
 
-  /// Finishes activating package [package] by saving [lockFile] in the cache.
-  void _writeLockFile(String dir, LockFile lockFile) {
-    writeTextFile(p.join(dir, 'pubspec.lock'), lockFile.serialize(null));
-  }
-
   /// Shows the user the currently active package with [name], if any.
   LockFile? _describeActive(String name, SystemCache cache) {
     late final LockFile lockFile;

lib/src/io.dart

@@ -172,7 +172,7 @@ List<int> readBinaryFile(String file) {
 }
 
 /// Reads the contents of the binary file [file] as a [Stream].
-Stream<List<int>> readBinaryFileAsSream(String file) {
+Stream<List<int>> readBinaryFileAsStream(String file) {
   log.io('Reading binary file $file.');
   var contents = File(file).openRead();
   return contents;