External hackages

lib/call-cabal-project-to-nix.nix

@@ -23,6 +23,7 @@
                      #     { "https://github.com/jgm/pandoc-citeproc"."0.17"
                      #         = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; }
                      #       ."${repo.location}"."${repo.tag}";
+, extra-hackage-tarballs ? []
 , ...
 }@args:
 # cabal-install versions before 2.4 will generate insufficient plan information.
@@ -41,7 +42,7 @@ let
       inherit src;
       filter = path: type:
         type == "directory" ||
-        pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; }
+        pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; }
     else src;
 
   # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when
@@ -225,7 +226,7 @@ let
     export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
     export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
     HOME=${dotCabal {
-      inherit cabal-install nix-tools;
+      inherit cabal-install nix-tools extra-hackage-tarballs;
       index-state =
         builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name))
           index-state-found;

mk-local-hackage-repo/default.nix

@@ -14,10 +14,10 @@
 # nix would be pointless as we'd have to hardcode them to produce the same output
 # reproducably.
 #
-{ pkgs, hackageTarball }:
-{ index-state, sha256, ... }@args:
-let index = hackageTarball args; in
-pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
+pkgs:
+{ name, index }:
+
+pkgs.runCommand "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } ''
 mkdir -p $out
 export expires="4000-01-01T00:00:00Z"
 

overlays/haskell.nix

@@ -70,13 +70,19 @@ self: super: {
             { pkg-def  # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix)
             , pkg-def-extras ? [] # Additional packages to augment the Base package set `pkg-def` with.
             , modules ? []
+            , extra-hackages ? [] # Extra Hackage repositories to use besides main one.
             }@args:
 
-            import ../package-set.nix (args // {
+            let
+              hackageAll = builtins.foldl' (base: extra: base // extra) hackage extra-hackages;
+            in
+
+            import ../package-set.nix {
+                inherit (args) pkg-def pkg-def-extras;
                 modules = defaultModules ++ modules;
                 pkgs = self;
-                inherit hackage pkg-def;
-            });
+                hackage = hackageAll;
+            };
 
         # Some boot packages (libiserv) are in lts, but not in hackage,
         # so we should not try to get it from hackage based on the stackage
@@ -128,6 +134,7 @@ self: super: {
             { plan-pkgs  # Path to the output of plan-to-nix
             , pkg-def-extras ? []
             , modules ? []
+            , extra-hackages ? []
             }@args:
 
             let
@@ -146,6 +153,7 @@ self: super: {
                 modules = [ { doExactConfig = true; } patchesModule ]
                        ++ modules
                        ++ plan-pkgs.modules or [];
+                inherit extra-hackages;
             };
 
         # Package sets for all stackage snapshots.
@@ -169,33 +177,60 @@ self: super: {
         nix-tools = self.buildPackages.haskell-nix.nix-tools-cross-compiled;
         # TODO perhaps there is a cleaner way to get a suitable nix-tools.
 
-            # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Produce a fixed output derivation from a moving target (hackage index tarball)
+        # Takes desired index-state and sha256 and produces a set { name, index }, where
+        # index points to "01-index.tar.gz" file downloaded from hackage.haskell.org.
         hackageTarball = { index-state, sha256, nix-tools ? self.haskell-nix.nix-tools, ... }:
             assert sha256 != null;
-            self.fetchurl {
-                name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}";
+            let at = builtins.replaceStrings [":"] [""] index-state; in
+            { name = "hackage.haskell.org-at-${at}";
+              index = self.fetchurl {
+                name = "01-index.tar.gz-at-${at}";
                 url = "https://hackage.haskell.org/01-index.tar.gz";
                 downloadToTemp = true;
                 postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}";
 
                 outputHashAlgo = "sha256";
                 outputHash = sha256;
+              };
             };
 
-        mkLocalHackageRepo = import ../mk-local-hackage-repo { inherit hackageTarball; pkgs = self; };
+        # Creates Cabal local repository from { name, index } set.
+        mkLocalHackageRepo = import ../mk-local-hackage-repo self;
 
-        dotCabal = { index-state, sha256, cabal-install, ... }@args:
-            self.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } ''
+        dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? [], ... }@args:
+            let
+              allTarballs = [ (hackageTarball args) ] ++ extra-hackage-tarballs;
+              allNames = self.lib.concatMapStringsSep "-" (tarball: tarball.name) allTarballs;
+              # Main Hackage index-state is embedded in its name and thus will propagate to
+              # dotCabalName anyway.
+              dotCabalName = "dot-cabal-" + allNames;
+            in
+            self.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; } ''
                 mkdir -p $out/.cabal
                 cat <<EOF > $out/.cabal/config
-                repository cached
-                    url: file:${mkLocalHackageRepo args}
-                    secure: True
-                    root-keys:
-                    key-threshold: 0
+                ${self.lib.concatStrings (
+                  map (tarball:
+                ''
+                repository ${tarball.name}
+                  url: file:${mkLocalHackageRepo tarball}
+                  secure: True
+                  root-keys:
+                  key-threshold: 0
+
+                '') allTarballs
+                )}
                 EOF
-                mkdir -p $out/.cabal/packages/cached
-                HOME=$out cabal new-update cached
+
+                # All repositories must be mkdir'ed before calling new-update on any repo,
+                # otherwise it fails.
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  mkdir -p $out/.cabal/packages/${name}
+                '') allTarballs)}
+
+                ${self.lib.concatStrings (map ({ name, ... }: ''
+                  HOME=$out cabal new-update ${name}
+                '') allTarballs)}
             '';
 
         # Some of features of haskell.nix rely on using a hackage index
@@ -429,6 +464,7 @@ self: super: {
                   pkg-def-extras = args.pkg-def-extras or [];
                   modules = (args.modules or [])
                           ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; };
+                  extra-hackages = args.extra-hackages or [];
                 };
             in { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; };
 

test/default.nix

@@ -175,6 +175,7 @@ let
     exe-only = callTest ./exe-only { inherit util; };
     stack-source-repo = callTest ./stack-source-repo {};
     lookup-sha256 = callTest ./lookup-sha256 {};
+    extra-hackage = callTest ./extra-hackage {};
 
     unit = unitTests;
   };

test/extra-hackage/README.md

@@ -0,0 +1,11 @@
+# Tests for extra Hackage functionality
+
+This directory contains two packages, `external-package-demo` and `external-package-user`, the
+second one depends on the first one. Both packages were created with `cabal init`.
+
+`external-package-demo` was uploaded to local Hackage at `localhost` and `01-index.tar.gz` from that
+Hackage was downloaded to this directory. Then the index file was processed with `hackage-to-nix`,
+the result is in `hackage/` directory.
+
+The tests check that `cabalProject'` is able to construct plan with dependencies from extra Hackage
+and then build the package itself.

test/extra-hackage/default.nix

@@ -0,0 +1,69 @@
+{ stdenv, cabalProject', haskellLib, recurseIntoAttrs, testSrc }:
+
+with stdenv.lib;
+
+let  
+
+  hackage = import ./hackage;
+
+  tarball = {
+    name = "extra-hackage-demo";
+    index = ./01-index.tar.gz;
+  };
+
+  demo-src = ./external-package-demo-0.1.0.0.tar.gz;
+
+  project = cabalProject' {
+    src = testSrc "extra-hackage/external-package-user";
+
+    extra-hackages = [ hackage ];
+    extra-hackage-tarballs = [ tarball ];
+
+    modules = [
+      # To prevent nix-build from trying to download it from the
+      # actual Hackage.
+      { packages.external-package-demo.src = demo-src; }
+    ];
+  };
+  packages = project.hsPkgs;
+
+in recurseIntoAttrs {
+  ifdInputs = {
+    inherit (project) plan-nix;
+  };
+  run = stdenv.mkDerivation {
+    name = "external-hackage-test";
+
+    buildCommand = ''
+      exe="${packages.external-package-user.components.exes.external-package-user}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
+      size=$(command stat --format '%s' "$exe")
+      printf "size of executable $exe is $size. \n" >& 2
+      # fixme: run on target platform when cross-compiled
+      printf "checking whether executable runs... " >& 2
+      cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user}
+    '' + (if stdenv.hostPlatform.isMusl
+      then ''
+        printf "checking that executable is statically linked... " >& 2
+        (ldd $exe 2>&1 || true) | grep -i "not a"
+      ''
+      else
+        # Skip this on aarch as we do not have an `ldd` tool
+        optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64) (''
+          printf "checking that executable is dynamically linked to system libraries... " >& 2
+        '' + optionalString stdenv.isLinux ''
+          ldd $exe | grep libpthread
+        '' + optionalString stdenv.isDarwin ''
+          otool -L $exe |grep .dylib
+      '')) + ''
+      printf "Checking that \"all\" component has the programs... " >& 2
+      all_exe="${packages.external-package-user.components.all}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}"
+      test -f "$all_exe"
+      echo "$all_exe" >& 2
+      touch $out
+    '';    
+    meta.platforms = platforms.all;
+    passthru = {
+      inherit project;
+    };  
+  };
+}

test/extra-hackage/external-package-demo/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Revision history for external-package-demo
+
+## 0.1.0.0 -- YYYY-mm-dd
+
+* First version. Released on an unsuspecting world.

test/extra-hackage/external-package-demo/Main.hs

@@ -0,0 +1,8 @@
+module Main where
+
+import qualified MyLib (someFunc)
+
+main :: IO ()
+main = do
+  putStrLn "Hello, Haskell!"
+  MyLib.someFunc

test/extra-hackage/external-package-demo/MyLib.hs

@@ -0,0 +1,4 @@
+module MyLib (someFunc) where
+
+someFunc :: IO ()
+someFunc = putStrLn "someFunc"

test/extra-hackage/external-package-demo/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

test/extra-hackage/external-package-demo/external-package-demo.cabal

@@ -0,0 +1,32 @@
+cabal-version:       2.4
+-- Initial package description 'external-package-demo.cabal' generated by
+-- 'cabal init'.  For further documentation, see
+-- http://haskell.org/cabal/users-guide/
+
+name:                external-package-demo
+version:             0.1.0.0
+synopsis:            Demo package.
+description:         Just a demo package...
+-- bug-reports:
+license:             BSD-3-Clause
+author:              Maxim Koltsov
+maintainer:          kolmax94@gmail.com
+-- copyright:
+-- category:
+extra-source-files:  CHANGELOG.md
+
+library
+  exposed-modules:     MyLib
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base < 5
+  -- hs-source-dirs:
+  default-language:    Haskell2010
+
+executable external-package-demo
+  main-is:             Main.hs
+  other-modules:       MyLib
+  -- other-extensions:
+  build-depends:       base, external-package-demo
+  -- hs-source-dirs:
+  default-language:    Haskell2010

test/extra-hackage/external-package-user/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Revision history for external-package-user
+
+## 0.1.0.0 -- YYYY-mm-dd
+
+* First version. Released on an unsuspecting world.

test/extra-hackage/external-package-user/Main.hs

@@ -0,0 +1,8 @@
+module Main where
+
+import qualified MyLib (someFunc)
+
+main :: IO ()
+main = do
+  putStrLn "Hello, Haskell!"
+  MyLib.someFunc

test/extra-hackage/external-package-user/MyLib.hs

@@ -0,0 +1,4 @@
+module MyLib (someFunc) where
+
+someFunc :: IO ()
+someFunc = putStrLn "someFunc"

test/extra-hackage/external-package-user/Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

test/extra-hackage/external-package-user/cabal.project

@@ -0,0 +1,4 @@
+packages: *.cabal
+
+repository local
+    url: http://127.0.0.1:7777

test/extra-hackage/external-package-user/external-package-user.cabal

@@ -0,0 +1,33 @@
+cabal-version:       2.4
+-- Initial package description 'external-package-user.cabal' generated by
+-- 'cabal init'.  For further documentation, see
+-- http://haskell.org/cabal/users-guide/
+
+name:                external-package-user
+version:             0.1.0.0
+synopsis:            User package
+description:         A user of external-package-demo
+-- bug-reports:
+license:             BSD-3-Clause
+author:              Maxim Koltsov
+maintainer:          kolmax94@gmail.com
+-- copyright:
+-- category:
+extra-source-files:  CHANGELOG.md
+
+library
+  exposed-modules:     MyLib
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       base < 5,
+                       external-package-demo
+  -- hs-source-dirs:
+  default-language:    Haskell2010
+
+executable external-package-user
+  main-is:             Main.hs
+  other-modules:       MyLib
+  -- other-extensions:
+  build-depends:       base, external-package-user
+  -- hs-source-dirs:
+  default-language:    Haskell2010

test/extra-hackage/hackage/default.nix

@@ -0,0 +1,9 @@
+with builtins; mapAttrs (_: mapAttrs (_: data: rec {
+ inherit (data) sha256;
+ revisions = (mapAttrs (rev: rdata: {
+  inherit (rdata) revNum sha256;
+  outPath = ./. + "/hackage/${rdata.outPath}";
+ }) data.revisions) // {
+  default = revisions."${data.revisions.default}";
+ };
+})) (fromJSON (readFile ./hackage.json))

test/extra-hackage/hackage/hackage.json

@@ -0,0 +1,15 @@
+{
+ "external-package-demo": {
+  "0.1.0.0": {
+   "revisions": {
+    "default": "r0",
+    "r0": {
+     "outPath": "external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix",
+     "revNum": 0,
+     "sha256": "3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7"
+    }
+   },
+   "sha256": "1fce0685dcb89200ed286b9ae0983322ebc8a2d5de0541da55a5b82797dba740"
+  }
+ }
+}