Add support for external Hackage repositories

Currently haskell.nix is not able to build Cabal projects that depend on
packages from private Hackage repositories, as it make only main Hackage
available to cabal. This is unfortunate.

This commit adds this functionality, by allowing the user to pass
`extra-hackages` and `extra-hackage-tarballs` to `mkPkgSet` and
`callCabalToNix` respectively, to add as much extra repositories as
needed.

This repositories are first made available to Cabal when calling
`v2-configure`, resulting in correct plans. Later they are combined with
global Hackage when building dependencies of the local packages.

Author: maksbotan

lib/call-cabal-project-to-nix.nix

@@ -16,6 +16,7 @@
                      # If the tests and benchmarks are not needed and they
                      # causes the wrong plan to be choosen, then we can use
                      # `configureArgs = "--disable-tests --disable-benchmarks";`
+, extra-hackage-tarballs ? []
 , ...
 }@args:
 # cabal-install versions before 2.4 will generate insufficient plan information.
@@ -201,7 +202,7 @@ let
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
     HOME=${dotCabal {
-      inherit cabal-install nix-tools;
+      inherit cabal-install nix-tools extra-hackage-tarballs;
       index-state =
         builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
           index-state-found;

mk-local-hackage-repo/default.nix

@@ -14,10 +14,10 @@
 # nix would be pointless as we'd have to hardcode them to produce the same output
 # reproducably.
 #
-{ pkgs, hackageTarball }:
-{ index-state, sha256, ... }@args:
-let index = hackageTarball args; in
-pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
+pkgs:
+{ name, index }:
+
+pkgs.runCommand "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
 mkdir -p $out
 export expires="4000-01-01T00:00:00Z"
 

overlays/haskell.nix

@@ -70,13 +70,19 @@ self: super: {
             { pkg-def  # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix)
             , pkg-def-extras ? [] # Additional packages to augment the Base package set `pkg-def` with.
             , modules ? []
+            , extra-hackages ? [] # Extra Hackage repositories to use besides main one.
             }@args:
 
-            import ../package-set.nix (args // {
+            let
+              hackageAll = builtins.foldl' (base: extra: base // extra) hackage extra-hackages;
+            in
+
+            import ../package-set.nix {
+                inherit (args) pkg-def pkg-def-extras;
                 modules = defaultModules ++ modules;
                 pkgs = self;
-                inherit hackage pkg-def;
-            });
+                hackage = hackageAll;
+            };
 
         # Some boot packages (libiserv) are in lts, but not in hackage,
         # so we should not try to get it from hackage based on the stackage
@@ -128,6 +134,7 @@ self: super: {
             { plan-pkgs  # Path to the output of plan-to-nix
             , pkg-def-extras ? []
             , modules ? []
+            , extra-hackages ? []
             }@args:
 
             let
@@ -146,6 +153,7 @@ self: super: {
                 modules = [ { doExactConfig = true; } patchesModule ]
                        ++ modules
                        ++ plan-pkgs.modules or [];
+                inherit extra-hackages;
             };
 
         # Package sets for all stackage snapshots.
@@ -169,33 +177,56 @@ self: super: {
         nix-tools = self.buildPackages.haskell-nix.nix-tools-cross-compiled;
         # TODO perhaps there is a cleaner way to get a suitable nix-tools.
 
-            # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Takes desired index-state and sha256 and produces a set { name, index }, where
+        # index points to "01-index.tar.gz" file downloaded from hackage.haskell.org.
         hackageTarball = { index-state, sha256, nix-tools ? self.haskell-nix.nix-tools, ... }:
             assert sha256 != null;
-            self.fetchurl {
-                name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}";
+            let at = builtins.replaceStrings [":"] [""] index-state; in
+            { name = "hackage.haskell.org-at-${at}";
+              index = self.fetchurl {
+                name = "01-index.tar.gz-at-${at}";
                 url = "https://hackage.haskell.org/01-index.tar.gz";
                 downloadToTemp = true;
                 postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
 
                 outputHashAlgo = "sha256";
                 outputHash = sha256;
+              };
             };
 
-        mkLocalHackageRepo = import ../mk-local-hackage-repo { inherit hackageTarball; pkgs = self; };
+        # Creates Cabal local repository from { name, index } set.
+        mkLocalHackageRepo = import ../mk-local-hackage-repo self;
 
-        dotCabal = { index-state, sha256, cabal-install, ... }@args:
+        dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? [], ... }@args:
+            let
+              allTarballs = [ (hackageTarball args) ] ++ extra-hackage-tarballs;
+            in
             self.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } ''
                 mkdir -p $out/.cabal
                 cat <<EOF > $out/.cabal/config
-                repository cached
-                    url: file:${mkLocalHackageRepo args}
-                    secure: True
-                    root-keys:
-                    key-threshold: 0
+                ${self.lib.concatStrings (
+                  map (tarball:
+                ''
+                repository ${tarball.name}
+                  url: file:${mkLocalHackageRepo tarball}
+                  secure: True
+                  root-keys:
+                  key-threshold: 0
+
+                '') allTarballs
+                )}
                 EOF
-                mkdir -p $out/.cabal/packages/cached
-                HOME=$out cabal new-update cached
+
+                # All repositories must be mkdir'ed before calling new-update on any repo,
+                # otherwise it fails.
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  mkdir -p $out/.cabal/packages/${name}
+                '') allTarballs)}
+
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  HOME=$out cabal new-update ${name}
+                '') allTarballs)}
             '';
 
         checkMaterialization = false; # This is the default. Use an overlay to set it to true and test all the materialized files
@@ -419,6 +450,7 @@ self: super: {
                   pkg-def-extras = args.pkg-def-extras or [];
                   modules = (args.modules or [])
                           ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; };
+                  extra-hackages = args.extra-hackages or [];
                 };
             in { inherit (pkg-set.config) hsPkgs; plan-nix = plan.nix; };