input-output-hk · hamishmack · Feb 17, 2022 · Feb 11, 2022 · Feb 11, 2022 · Feb 11, 2022
diff --git a/changelog.md b/changelog.md
@@ -1,6 +1,13 @@
 This file contains a summary of changes to Haskell.nix and `nix-tools`
 that will impact users.
 
+## Feb 16, 2022
+* Removed lookupSha256 argument from project functions.
+  Pass a `sha256map` instead.
+* Added better support for `repository` in `cabal.project`.  These
+  blocks should now work without the need for passing `extra-hackages` and
+  `extra-hackage-tarballs`.
+
 ## Aug 6, 2021
 * Included dependencies of haskell.nix that were tracked in `nix/sources.json`
   as flake inputs (`flake.lock` replaces `nix/sources.json`).

diff --git a/lib/cabal-project-parser.nix b/lib/cabal-project-parser.nix
@@ -74,17 +74,21 @@ let
   #   --shar256: 003lm3pm0000hbfmii7xcdd9v20000flxf7gdl2pyxia7p014i8z
   # will be trated like a field and returned here
   # (used in call-cabal-project-to-nix.nix to create a fixed-output derivation)
-  extractRepoData = cabalProjectFileName: lookupSha256: repo: {
+  extractSourceRepoPackageData = cabalProjectFileName: sha256map: repo: {
     url = repo.location;
     ref = repo.tag;
-    sha256 = repo."--sha256" or (lookupSha256 repo);
+    sha256 = repo."--sha256" or (
+      if sha256map != null
+        then sha256map."${repo.location}"."${repo.tag}"
+        else null);
     subdirs = if repo ? subdir
       then pkgs.lib.filter (x: x != "") (pkgs.lib.splitString " " repo.subdir)
       else ["."];
   };
 
   # Parse a source-repository-package and return data of `type: git` repositories
-  parseBlock = cabalProjectFileName: lookupSha256: block:
+  # See tests/unit.nix for examples of input and output.
+  parseSourceRepositoryPackageBlock = cabalProjectFileName: sha256map: block:
     let
       x = span (pkgs.lib.strings.hasPrefix " ") (pkgs.lib.splitString "\n" block);
       attrs = parseBlockLines x.fst;
@@ -95,10 +99,106 @@ let
           otherText = "\nsource-repository-package\n" + block;
         }
         else {
-          sourceRepo = extractRepoData cabalProjectFileName lookupSha256 attrs;
+          sourceRepo = extractSourceRepoPackageData cabalProjectFileName sha256map attrs;
           otherText = pkgs.lib.strings.concatStringsSep "\n" x.snd;
         };
 
+  parseSourceRepositoryPackages = cabalProjectFileName: sha256map: source-repo-override: projectFile:
+    let
+      blocks = pkgs.lib.splitString "\nsource-repository-package\n" ("\n" + projectFile);
+      initialText = pkgs.lib.lists.take 1 blocks;
+      repoBlocks = builtins.map (parseSourceRepositoryPackageBlock cabalProjectFileName sha256map) (pkgs.lib.lists.drop 1 blocks);
+      overrideSourceRepo = sourceRepo: (source-repo-override.${sourceRepo.url} or (pkgs.lib.id)) sourceRepo;
+    in {
+      sourceRepos = pkgs.lib.lists.map (block: overrideSourceRepo block.sourceRepo) repoBlocks;
+      otherText = pkgs.lib.strings.concatStringsSep "\n" (
+        initialText
+        ++ (builtins.map (x: x.otherText) repoBlocks));
+    };
+
+  # Parse and replace repository
+  # This works in a similar way to the `source-repository-package` but we are
+  # able to simply replace the `repository` blocks with local `file:/nix/store` ones.
+  # This works because `cabal configure` does not include any of the `/nix/sore/`
+  # paths in the `plan.json` (so materialized plan-nix will still work as expeced).
+  # See tests/unit.nix for examples of input and output.
+  parseRepositoryBlock = cabalProjectFileName: sha256map: cabal-install: nix-tools: block:
+    let
+      lines = pkgs.lib.splitString "\n" block;
+      # The first line will contain the repository name.
+      x = span (pkgs.lib.strings.hasPrefix " ") (__tail lines);
+      attrs = parseBlockLines x.fst;
+      sha256 = attrs."--sha256" or (
+        if sha256map != null
+          then sha256map."${attrs.url}"
+          else null);
+    in rec {
+      # This is `some-name` from the `repository some-name` line in the `cabal.project` file.
+      name = __head lines;
+      # The $HOME dir with `.cabal` sub directory after running `cabal new-update` to download the repository
+      home =
+        pkgs.evalPackages.runCommandLocal name ({
+          nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
+          LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
+          LANG = "en_US.UTF-8";
+        } // pkgs.lib.optionalAttrs (sha256 != null) {
+          outputHashMode = "recursive";
+          outputHashAlgo = "sha256";
+          outputHash = sha256;
+        }) ''
+            mkdir -p $out/.cabal
+            cat <<EOF > $out/.cabal/config
+            repository ${name}
+              url: ${attrs.url}
+              ${pkgs.lib.optionalString (attrs ? secure) "secure: ${attrs.secure}"}
+              ${pkgs.lib.optionalString (attrs ? root-keys) "root-keys: ${attrs.root-keys}"}
+              ${pkgs.lib.optionalString (attrs ? key-threshold) "key-threshold: ${attrs.key-threshold}"}
+            EOF
+
+            export SSL_CERT_FILE=${pkgs.evalPackages.cacert}/etc/ssl/certs/ca-bundle.crt
+            mkdir -p $out/.cabal/packages/${name}
+            HOME=$out cabal new-update ${name}
+        '';
+      # Output of hackage-to-nix
+      hackage = import (
+        pkgs.evalPackages.runCommandLocal ("hackage-to-nix-" + name) {
+          nativeBuildInputs = [ cabal-install pkgs.evalPackages.curl nix-tools ];
+          LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
+          LANG = "en_US.UTF-8";
+        } ''
+          mkdir -p $out
+          hackage-to-nix $out ${home}/.cabal/packages/${name}/01-index.tar ${attrs.url}
+        '');
+      # Tarball info in the same format as `extra-hackage-tarballs` passed in to cabalProject
+      tarball = {
+        ${name} = home + "/.cabal/packages/${name}/01-index.tar.gz";
+      };
+      # Replacement `repository` block using `file:` uri and the remaining text (text that was not part of the attribute block).
+      updatedText = ''
+        repository ${name}
+          url: file:${home + "/.cabal/packages/${name}"}
+          secure: True
+          root-keys:
+          key-threshold: 0
+        '' + pkgs.lib.strings.concatStringsSep "\n" x.snd;
+    };
+
+  parseRepositories = cabalProjectFileName: sha256map: cabal-install: nix-tools: projectFile:
+    let
+      # This will leave the name of repository in the first line of each block
+      blocks = pkgs.lib.splitString "\nrepository " ("\n" + projectFile);
+      initialText = pkgs.lib.lists.take 1 blocks;
+      repoBlocks = builtins.map (parseRepositoryBlock cabalProjectFileName sha256map cabal-install nix-tools) (pkgs.lib.lists.drop 1 blocks);
+    in {
+      extra-hackages = pkgs.lib.lists.map (block: block.hackage) repoBlocks;
+      tarballs = pkgs.lib.lists.foldl' (x: block: x // block.tarball) {} repoBlocks;
+      updatedText = pkgs.lib.strings.concatStringsSep "\n" (
+        initialText
+        ++ (builtins.map (x: x.updatedText) repoBlocks));
+    };
+
 in {
-  inherit parseIndexState parseBlockLines parseBlock;
+  inherit parseIndexState parseSourceRepositoryPackages parseRepositories
+    # These are only exposed for tests
+    parseSourceRepositoryPackageBlock parseRepositoryBlock;
 }
diff --git a/lib/call-cabal-project-to-nix.nix b/lib/call-cabal-project-to-nix.nix
@@ -30,13 +30,14 @@ in
                      # An alternative to adding `--sha256` comments into the
                      # cabal.project file:
                      #   sha256map =
-                     #     { "https://github.com/jgm/pandoc-citeproc"."0.17"
-                     #         = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
-, lookupSha256  ?
-  if sha256map != null
-    then { location, tag, ...}: sha256map."${location}"."${tag}"
-    else _: null
-, extra-hackage-tarballs ? []
+                     #     { # For a `source-repository-package` use the `location` and `tag` as the key
+                     #       "https://github.com/jgm/pandoc-citeproc"."0.17"
+                     #         = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q";
+                     #       # For a `repository` use the `url` as the key
+                     #       "https://raw.githubusercontent.com/input-output-hk/hackage-overlay-ghcjs/bfc363b9f879c360e0a0460ec0c18ec87222ec32"
+                     #         = "sha256-g9xGgJqYmiczjxjQ5JOiK5KUUps+9+nlNGI/0SpSOpg=";
+                     #     };
+, extra-hackage-tarballs ? {}
 , source-repo-override ? {} # Cabal seems to behave incoherently when
                             # two source-repository-package entries
                             # provide the same packages, making it
@@ -192,15 +193,15 @@ let
 
   replaceSourceRepos = projectFile:
     let
-      fetchRepo = fetchgit: repoData:
+      fetchPackageRepo = fetchgit: repoData:
         let
           fetched =
             if repoData.sha256 != null
             then fetchgit { inherit (repoData) url sha256; rev = repoData.ref; }
             else
               let drv = builtins.fetchGit { inherit (repoData) url ref; };
               in __trace "WARNING: No sha256 found for source-repository-package ${repoData.url} ${repoData.ref} download may fail in restricted mode (hydra)"
-                (__trace "Consider adding `--sha256: ${hashPath drv}` to the ${cabalProjectFileName} file or passing in a lookupSha256 argument"
+                (__trace "Consider adding `--sha256: ${hashPath drv}` to the ${cabalProjectFileName} file or passing in a sha256map argument"
                  drv);
         in {
           # Download the source-repository-package commit and add it to a minimal git
@@ -221,14 +222,13 @@ let
           tag = "minimal";
         };
 
-      blocks = pkgs.lib.splitString "\nsource-repository-package\n" ("\n" + projectFile);
-      initialText = pkgs.lib.lists.take 1 blocks;
-      repoBlocks = builtins.map (pkgs.haskell-nix.haskellLib.parseBlock cabalProjectFileName lookupSha256) (pkgs.lib.lists.drop 1 blocks);
-      overrideSourceRepo = sourceRepo: (source-repo-override.${sourceRepo.url} or (pkgs.lib.id)) sourceRepo;
-      sourceRepoData = pkgs.lib.lists.map (x: overrideSourceRepo x.sourceRepo) repoBlocks;
-      otherText = pkgs.evalPackages.writeText "cabal.project" (pkgs.lib.strings.concatStringsSep "\n" (
-        initialText
-        ++ (builtins.map (x: x.otherText) repoBlocks)));
+      # Parse the `source-repository-package` blocks
+      sourceRepoPackageResult = pkgs.haskell-nix.haskellLib.parseSourceRepositoryPackages
+        cabalProjectFileName sha256map source-repo-override projectFile;
+
+      # Parse the `repository` blocks
+      repoResult = pkgs.haskell-nix.haskellLib.parseRepositories
+        cabalProjectFileName sha256map cabal-install nix-tools sourceRepoPackageResult.otherText;
 
       # we need the repository content twice:
       # * at eval time (below to build the fixed project file)
@@ -239,12 +239,13 @@ let
       #   on the target system would use, so that the derivation is unaffected
       #   and, say, a linux release build job can identify the derivation
       #   as built by a darwin builder, and fetch it from a cache
-      sourceReposEval = builtins.map (fetchRepo pkgs.evalPackages.fetchgit) sourceRepoData;
-      sourceReposBuild = builtins.map (x: (fetchRepo pkgs.fetchgit x).fetched) sourceRepoData;
+      sourceReposEval = builtins.map (fetchPackageRepo pkgs.evalPackages.fetchgit) sourceRepoPackageResult.sourceRepos;
+      sourceReposBuild = builtins.map (x: (fetchPackageRepo pkgs.fetchgit x).fetched) sourceRepoPackageResult.sourceRepos;
     in {
       sourceRepos = sourceReposBuild;
+      inherit (repoResult) tarballs extra-hackages;
       makeFixedProjectFile = ''
-        cp -f ${otherText} ./cabal.project
+        cp -f ${pkgs.evalPackages.writeText "cabal.project" repoResult.updatedText} ./cabal.project
       '' +
         pkgs.lib.optionalString (builtins.length sourceReposEval != 0) (''
         chmod +w -R ./cabal.project
@@ -279,7 +280,7 @@ let
 
   fixedProject =
     if rawCabalProject == null
-      then { sourceRepos = []; makeFixedProjectFile = ""; replaceLocations = ""; }
+      then { sourceRepos = []; tarballs = {}; extra-hackages = []; makeFixedProjectFile = ""; replaceLocations = ""; }
       else replaceSourceRepos rawCabalProject;
 
   # The use of the actual GHC can cause significant problems:
@@ -524,7 +525,8 @@ let
       # some packages that will be excluded by `index-state-found`
       # which is used by cabal (cached-index-state >= index-state-found).
       dotCabal {
-        inherit cabal-install nix-tools extra-hackage-tarballs;
+        inherit cabal-install nix-tools;
+        extra-hackage-tarballs = fixedProject.tarballs // extra-hackage-tarballs;
         index-state = cached-index-state;
         sha256 = index-sha256-found;
       }
@@ -610,5 +612,5 @@ in {
   projectNix = plan-nix;
   index-state = index-state-found;
   inherit src;
-  inherit (fixedProject) sourceRepos;
+  inherit (fixedProject) sourceRepos extra-hackages;
 }
diff --git a/lib/default.nix b/lib/default.nix
@@ -275,7 +275,7 @@ in {
 
   inherit (import ./cabal-project-parser.nix {
     inherit pkgs;
-  }) parseIndexState parseBlock;
+  }) parseIndexState parseSourceRepositoryPackages parseRepositories parseSourceRepositoryPackageBlock parseRepositoryBlock;
 
 
   cabalToNixpkgsLicense = import ./spdx/cabal.nix pkgs;

diff --git a/lib/stack-cache-generator.nix b/lib/stack-cache-generator.nix
@@ -9,10 +9,6 @@
                      #   sha256map =
                      #     { "https://github.com/jgm/pandoc-citeproc"."0.17"
                      #         = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
-, lookupSha256 ?
-  if sha256map != null
-    then { location, tag, ...}: sha256map."${location}"."${tag}"
-    else _: null
 , branchMap    ? null
                      # A way to specify in which branch a git commit can
                      # be found
@@ -85,12 +81,10 @@ in with pkgs.lib;
 concatMap (dep:
         let
             is-private = private dep.url;
-            sha256 = if dep.sha256 != null
-              then dep.sha256
-              else lookupSha256 {
-                  location = dep.url;
-                  tag = dep.rev;
-                };
+            sha256 =
+              if dep.sha256 != null then dep.sha256
+              else if sha256map != null then { location, tag, ...}: sha256map."${dep.url}"."${dep.rev}"
+              else null;
             branch = lookupBranch {
               location = dep.url;
               tag = dep.rev;

diff --git a/mk-local-hackage-repo/default.nix b/mk-local-hackage-repo/default.nix
@@ -17,13 +17,13 @@
 pkgs:
 { name, index }:
 
-pkgs.evalPackages.runCommandLocal "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.evalPackages.nix ]; } ''
+pkgs.evalPackages.runCommandLocal "hackage-repo-${name}" {} ''
 mkdir -p $out
 export expires="4000-01-01T00:00:00Z"
 
 ln -sf ${index} $out/01-index.tar.gz
-export index_md5=$(nix-hash --flat --type md5 ${index})
-export index_sha256=$(nix-hash --flat --type sha256 ${index})
+export index_md5=$(md5sum ${index} | awk '{ print $1 }')
+export index_sha256=$(sha256sum ${index} | awk '{ print $1 }')
 ${
   # When possible check the hash we calculate here against the `outputHash`
   # of the index derivation (when the `extra-hackages` feature is used the index
@@ -37,18 +37,18 @@ ${
 export index_length=$(stat --printf="%s" ${index})
 
 substituteAll ${./root.json} $out/root.json
-export root_md5=$(nix-hash --flat --type md5 $out/root.json)
-export root_sha256=$(nix-hash --flat --type sha256 $out/root.json)
+export root_md5=$(md5sum $out/root.json | awk '{ print $1 }')
+export root_sha256=$(sha256sum $out/root.json | awk '{ print $1 }')
 export root_length=$(stat --printf="%s" $out/root.json)
 
 substituteAll ${./mirrors.json} $out/mirrors.json
-export mirrors_md5=$(nix-hash --flat --type md5 $out/mirrors.json)
-export mirrors_sha256=$(nix-hash --flat --type sha256 $out/mirrors.json)
+export mirrors_md5=$(md5sum $out/mirrors.json | awk '{ print $1 }')
+export mirrors_sha256=$(sha256sum $out/mirrors.json | awk '{ print $1 }')
 export mirrors_length=$(stat --printf="%s" $out/mirrors.json)
 
 substituteAll ${./snapshot.json} $out/snapshot.json
-export snapshot_md5=$(nix-hash --flat --type md5 $out/snapshot.json)
-export snapshot_sha256=$(nix-hash --flat --type sha256 $out/snapshot.json)
+export snapshot_md5=$(md5sum $out/snapshot.json | awk '{ print $1 }')
+export snapshot_sha256=$(sha256sum $out/snapshot.json | awk '{ print $1 }')
 export snapshot_length=$(stat --printf="%s" $out/snapshot.json)
 
 substituteAll ${./timestamp.json} $out/timestamp.json

diff --git a/modules/cabal-project.nix b/modules/cabal-project.nix
@@ -105,15 +105,9 @@ in {
               = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
       '';
     };
-    lookupSha256 = mkOption {
-      type = nullOr unspecified;
-      default = if config.sha256map != null
-        then { location, tag, ...}: config.sha256map.${location}.${tag}
-        else _: null;
-    };
     extra-hackage-tarballs = mkOption {
-      type = nullOr (listOf unspecified);
-      default = [];
+      type = nullOr attrs;
+      default = {};
     };
     source-repo-override = mkOption {
       type = attrsOf (functionTo attrs);

diff --git a/modules/stack-project.nix b/modules/stack-project.nix
@@ -72,12 +72,6 @@ with types;
               = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
       '';
     };
-    lookupSha256 = mkOption {
-      type = nullOr unspecified;
-      default = if config.sha256map != null
-        then { location, tag, ...}: config.sha256map.${location}.${tag}
-        else _: null;
-    };
     branchMap = mkOption {
       type = nullOr unspecified;
       default = null;

diff --git a/overlays/hackage-quirks.nix b/overlays/hackage-quirks.nix
@@ -43,10 +43,9 @@ in { haskell-nix = prev.haskell-nix // {
     pandoc = {
       # Function that returns a sha256 string by looking up the location
       # and tag in a nested attrset
-      lookupSha256 = { location, tag, ... }:
+      sha256map =
         { "https://github.com/jgm/pandoc-citeproc"."0.17"
-            = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
-          ."${location}"."${tag}";
+            = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; };
     };
 
     # See https://github.com/input-output-hk/haskell.nix/issues/948