diff --git a/lib/call-cabal-project-to-nix.nix b/lib/call-cabal-project-to-nix.nix index 45bd6ccebf..2e89473478 100644 --- a/lib/call-cabal-project-to-nix.nix +++ b/lib/call-cabal-project-to-nix.nix @@ -23,6 +23,7 @@ # { "https://github.com/jgm/pandoc-citeproc"."0.17" # = "0dxx8cp2xndpw3jwiawch2dkrkp15mil7pyx7dvd810pwc22pm2q"; } # ."${repo.location}"."${repo.tag}"; +, extra-hackage-tarballs ? [] , ... }@args: # cabal-install versions before 2.4 will generate insufficient plan information. @@ -41,7 +42,7 @@ let inherit src; filter = path: type: type == "directory" || - pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" "package.yaml" ]; } + pkgs.lib.any (i: (pkgs.lib.hasSuffix i path)) [ ".project" ".cabal" ".freeze" "package.yaml" ]; } else src; # Using origSrcSubDir bypasses any cleanSourceWith so that it will work when @@ -225,7 +226,7 @@ let export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt HOME=${dotCabal { - inherit cabal-install nix-tools; + inherit cabal-install nix-tools extra-hackage-tarballs; index-state = builtins.trace ("Using index-state: ${index-state-found}" + (if name == null then "" else " for " + name)) index-state-found; diff --git a/mk-local-hackage-repo/default.nix b/mk-local-hackage-repo/default.nix index b2ea5a4d52..055cc0d674 100644 --- a/mk-local-hackage-repo/default.nix +++ b/mk-local-hackage-repo/default.nix @@ -14,10 +14,10 @@ # nix would be pointless as we'd have to hardcode them to produce the same output # reproducably. # -{ pkgs, hackageTarball }: -{ index-state, sha256, ... }@args: -let index = hackageTarball args; in -pkgs.runCommand "hackage-repo-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } '' +pkgs: +{ name, index }: + +pkgs.runCommand "hackage-repo-${name}" { nativeBuildInputs = [ pkgs.buildPackages.nix ]; } '' mkdir -p $out export expires="4000-01-01T00:00:00Z" diff --git a/overlays/haskell.nix b/overlays/haskell.nix index 8f53759456..3a752c2775 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -70,13 +70,19 @@ self: super: { { pkg-def # Base package set. Either from stackage (via stack-to-nix) or from a cabal projects plan file (via plan-to-nix) , pkg-def-extras ? [] # Additional packages to augment the Base package set `pkg-def` with. , modules ? [] + , extra-hackages ? [] # Extra Hackage repositories to use besides main one. }@args: - import ../package-set.nix (args // { + let + hackageAll = builtins.foldl' (base: extra: base // extra) hackage extra-hackages; + in + + import ../package-set.nix { + inherit (args) pkg-def pkg-def-extras; modules = defaultModules ++ modules; pkgs = self; - inherit hackage pkg-def; - }); + hackage = hackageAll; + }; # Some boot packages (libiserv) are in lts, but not in hackage, # so we should not try to get it from hackage based on the stackage @@ -128,6 +134,7 @@ self: super: { { plan-pkgs # Path to the output of plan-to-nix , pkg-def-extras ? [] , modules ? [] + , extra-hackages ? [] }@args: let @@ -146,6 +153,7 @@ self: super: { modules = [ { doExactConfig = true; } patchesModule ] ++ modules ++ plan-pkgs.modules or []; + inherit extra-hackages; }; # Package sets for all stackage snapshots. @@ -169,33 +177,60 @@ self: super: { nix-tools = self.buildPackages.haskell-nix.nix-tools-cross-compiled; # TODO perhaps there is a cleaner way to get a suitable nix-tools. - # Produce a fixed output derivation from a moving target (hackage index tarball) + # Produce a fixed output derivation from a moving target (hackage index tarball) + # Takes desired index-state and sha256 and produces a set { name, index }, where + # index points to "01-index.tar.gz" file downloaded from hackage.haskell.org. hackageTarball = { index-state, sha256, nix-tools ? self.haskell-nix.nix-tools, ... }: assert sha256 != null; - self.fetchurl { - name = "01-index.tar.gz-at-${builtins.replaceStrings [":"] [""] index-state}"; + let at = builtins.replaceStrings [":"] [""] index-state; in + { name = "hackage.haskell.org-at-${at}"; + index = self.fetchurl { + name = "01-index.tar.gz-at-${at}"; url = "https://hackage.haskell.org/01-index.tar.gz"; downloadToTemp = true; postFetch = "${nix-tools}/bin/truncate-index -o $out -i $downloadedFile -s ${index-state}"; outputHashAlgo = "sha256"; outputHash = sha256; + }; }; - mkLocalHackageRepo = import ../mk-local-hackage-repo { inherit hackageTarball; pkgs = self; }; + # Creates Cabal local repository from { name, index } set. + mkLocalHackageRepo = import ../mk-local-hackage-repo self; - dotCabal = { index-state, sha256, cabal-install, ... }@args: - self.runCommand "dot-cabal-at-${builtins.replaceStrings [":"] [""] index-state}" { nativeBuildInputs = [ cabal-install ]; } '' + dotCabal = { index-state, sha256, cabal-install, extra-hackage-tarballs ? [], ... }@args: + let + allTarballs = [ (hackageTarball args) ] ++ extra-hackage-tarballs; + allNames = self.lib.concatMapStringsSep "-" (tarball: tarball.name) allTarballs; + # Main Hackage index-state is embedded in its name and thus will propagate to + # dotCabalName anyway. + dotCabalName = "dot-cabal-" + allNames; + in + self.runCommand dotCabalName { nativeBuildInputs = [ cabal-install ]; } '' mkdir -p $out/.cabal cat < $out/.cabal/config - repository cached - url: file:${mkLocalHackageRepo args} - secure: True - root-keys: - key-threshold: 0 + ${self.lib.concatStrings ( + map (tarball: + '' + repository ${tarball.name} + url: file:${mkLocalHackageRepo tarball} + secure: True + root-keys: + key-threshold: 0 + + '') allTarballs + )} EOF - mkdir -p $out/.cabal/packages/cached - HOME=$out cabal new-update cached + + # All repositories must be mkdir'ed before calling new-update on any repo, + # otherwise it fails. + ${self.lib.concatStrings (map ({ name, ... }: '' + mkdir -p $out/.cabal/packages/${name} + '') allTarballs)} + + ${self.lib.concatStrings (map ({ name, ... }: '' + HOME=$out cabal new-update ${name} + '') allTarballs)} ''; # Some of features of haskell.nix rely on using a hackage index @@ -429,6 +464,7 @@ self: super: { pkg-def-extras = args.pkg-def-extras or []; modules = (args.modules or []) ++ self.lib.optional (args ? ghc) { ghc.package = args.ghc; }; + extra-hackages = args.extra-hackages or []; }; in { inherit (pkg-set.config) hsPkgs; inherit pkg-set; plan-nix = plan.nix; }; diff --git a/test/default.nix b/test/default.nix index 82b978eaa0..2a80a20c3e 100644 --- a/test/default.nix +++ b/test/default.nix @@ -175,6 +175,7 @@ let exe-only = callTest ./exe-only { inherit util; }; stack-source-repo = callTest ./stack-source-repo {}; lookup-sha256 = callTest ./lookup-sha256 {}; + extra-hackage = callTest ./extra-hackage {}; unit = unitTests; }; diff --git a/test/extra-hackage/01-index.tar.gz b/test/extra-hackage/01-index.tar.gz new file mode 100644 index 0000000000..d1b4cee2cc Binary files /dev/null and b/test/extra-hackage/01-index.tar.gz differ diff --git a/test/extra-hackage/README.md b/test/extra-hackage/README.md new file mode 100644 index 0000000000..22de64e40a --- /dev/null +++ b/test/extra-hackage/README.md @@ -0,0 +1,11 @@ +# Tests for extra Hackage functionality + +This directory contains two packages, `external-package-demo` and `external-package-user`, the +second one depends on the first one. Both packages were created with `cabal init`. + +`external-package-demo` was uploaded to local Hackage at `localhost` and `01-index.tar.gz` from that +Hackage was downloaded to this directory. Then the index file was processed with `hackage-to-nix`, +the result is in `hackage/` directory. + +The tests check that `cabalProject'` is able to construct plan with dependencies from extra Hackage +and then build the package itself. diff --git a/test/extra-hackage/default.nix b/test/extra-hackage/default.nix new file mode 100644 index 0000000000..b3a7c2a783 --- /dev/null +++ b/test/extra-hackage/default.nix @@ -0,0 +1,69 @@ +{ stdenv, cabalProject', haskellLib, recurseIntoAttrs, testSrc }: + +with stdenv.lib; + +let + + hackage = import ./hackage; + + tarball = { + name = "extra-hackage-demo"; + index = ./01-index.tar.gz; + }; + + demo-src = ./external-package-demo-0.1.0.0.tar.gz; + + project = cabalProject' { + src = testSrc "extra-hackage/external-package-user"; + + extra-hackages = [ hackage ]; + extra-hackage-tarballs = [ tarball ]; + + modules = [ + # To prevent nix-build from trying to download it from the + # actual Hackage. + { packages.external-package-demo.src = demo-src; } + ]; + }; + packages = project.hsPkgs; + +in recurseIntoAttrs { + ifdInputs = { + inherit (project) plan-nix; + }; + run = stdenv.mkDerivation { + name = "external-hackage-test"; + + buildCommand = '' + exe="${packages.external-package-user.components.exes.external-package-user}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}" + size=$(command stat --format '%s' "$exe") + printf "size of executable $exe is $size. \n" >& 2 + # fixme: run on target platform when cross-compiled + printf "checking whether executable runs... " >& 2 + cat ${haskellLib.check packages.external-package-user.components.exes.external-package-user} + '' + (if stdenv.hostPlatform.isMusl + then '' + printf "checking that executable is statically linked... " >& 2 + (ldd $exe 2>&1 || true) | grep -i "not a" + '' + else + # Skip this on aarch as we do not have an `ldd` tool + optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64) ('' + printf "checking that executable is dynamically linked to system libraries... " >& 2 + '' + optionalString stdenv.isLinux '' + ldd $exe | grep libpthread + '' + optionalString stdenv.isDarwin '' + otool -L $exe |grep .dylib + '')) + '' + printf "Checking that \"all\" component has the programs... " >& 2 + all_exe="${packages.external-package-user.components.all}/bin/external-package-user${stdenv.hostPlatform.extensions.executable}" + test -f "$all_exe" + echo "$all_exe" >& 2 + touch $out + ''; + meta.platforms = platforms.all; + passthru = { + inherit project; + }; + }; +} diff --git a/test/extra-hackage/external-package-demo-0.1.0.0.tar.gz b/test/extra-hackage/external-package-demo-0.1.0.0.tar.gz new file mode 100644 index 0000000000..5ad2e497a5 Binary files /dev/null and b/test/extra-hackage/external-package-demo-0.1.0.0.tar.gz differ diff --git a/test/extra-hackage/external-package-demo/CHANGELOG.md b/test/extra-hackage/external-package-demo/CHANGELOG.md new file mode 100644 index 0000000000..adb53d37a1 --- /dev/null +++ b/test/extra-hackage/external-package-demo/CHANGELOG.md @@ -0,0 +1,5 @@ +# Revision history for external-package-demo + +## 0.1.0.0 -- YYYY-mm-dd + +* First version. Released on an unsuspecting world. diff --git a/test/extra-hackage/external-package-demo/Main.hs b/test/extra-hackage/external-package-demo/Main.hs new file mode 100644 index 0000000000..60d904e8c1 --- /dev/null +++ b/test/extra-hackage/external-package-demo/Main.hs @@ -0,0 +1,8 @@ +module Main where + +import qualified MyLib (someFunc) + +main :: IO () +main = do + putStrLn "Hello, Haskell!" + MyLib.someFunc diff --git a/test/extra-hackage/external-package-demo/MyLib.hs b/test/extra-hackage/external-package-demo/MyLib.hs new file mode 100644 index 0000000000..e657c4403f --- /dev/null +++ b/test/extra-hackage/external-package-demo/MyLib.hs @@ -0,0 +1,4 @@ +module MyLib (someFunc) where + +someFunc :: IO () +someFunc = putStrLn "someFunc" diff --git a/test/extra-hackage/external-package-demo/Setup.hs b/test/extra-hackage/external-package-demo/Setup.hs new file mode 100644 index 0000000000..9a994af677 --- /dev/null +++ b/test/extra-hackage/external-package-demo/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/test/extra-hackage/external-package-demo/external-package-demo.cabal b/test/extra-hackage/external-package-demo/external-package-demo.cabal new file mode 100644 index 0000000000..b41df187df --- /dev/null +++ b/test/extra-hackage/external-package-demo/external-package-demo.cabal @@ -0,0 +1,32 @@ +cabal-version: 2.4 +-- Initial package description 'external-package-demo.cabal' generated by +-- 'cabal init'. For further documentation, see +-- http://haskell.org/cabal/users-guide/ + +name: external-package-demo +version: 0.1.0.0 +synopsis: Demo package. +description: Just a demo package... +-- bug-reports: +license: BSD-3-Clause +author: Maxim Koltsov +maintainer: kolmax94@gmail.com +-- copyright: +-- category: +extra-source-files: CHANGELOG.md + +library + exposed-modules: MyLib + -- other-modules: + -- other-extensions: + build-depends: base < 5 + -- hs-source-dirs: + default-language: Haskell2010 + +executable external-package-demo + main-is: Main.hs + other-modules: MyLib + -- other-extensions: + build-depends: base, external-package-demo + -- hs-source-dirs: + default-language: Haskell2010 diff --git a/test/extra-hackage/external-package-user/CHANGELOG.md b/test/extra-hackage/external-package-user/CHANGELOG.md new file mode 100644 index 0000000000..1b3a4d2fbd --- /dev/null +++ b/test/extra-hackage/external-package-user/CHANGELOG.md @@ -0,0 +1,5 @@ +# Revision history for external-package-user + +## 0.1.0.0 -- YYYY-mm-dd + +* First version. Released on an unsuspecting world. diff --git a/test/extra-hackage/external-package-user/Main.hs b/test/extra-hackage/external-package-user/Main.hs new file mode 100644 index 0000000000..60d904e8c1 --- /dev/null +++ b/test/extra-hackage/external-package-user/Main.hs @@ -0,0 +1,8 @@ +module Main where + +import qualified MyLib (someFunc) + +main :: IO () +main = do + putStrLn "Hello, Haskell!" + MyLib.someFunc diff --git a/test/extra-hackage/external-package-user/MyLib.hs b/test/extra-hackage/external-package-user/MyLib.hs new file mode 100644 index 0000000000..e657c4403f --- /dev/null +++ b/test/extra-hackage/external-package-user/MyLib.hs @@ -0,0 +1,4 @@ +module MyLib (someFunc) where + +someFunc :: IO () +someFunc = putStrLn "someFunc" diff --git a/test/extra-hackage/external-package-user/Setup.hs b/test/extra-hackage/external-package-user/Setup.hs new file mode 100644 index 0000000000..9a994af677 --- /dev/null +++ b/test/extra-hackage/external-package-user/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/test/extra-hackage/external-package-user/cabal.project b/test/extra-hackage/external-package-user/cabal.project new file mode 100644 index 0000000000..013aeca603 --- /dev/null +++ b/test/extra-hackage/external-package-user/cabal.project @@ -0,0 +1,4 @@ +packages: *.cabal + +repository local + url: http://127.0.0.1:7777 diff --git a/test/extra-hackage/external-package-user/external-package-user.cabal b/test/extra-hackage/external-package-user/external-package-user.cabal new file mode 100644 index 0000000000..92a52cd5fe --- /dev/null +++ b/test/extra-hackage/external-package-user/external-package-user.cabal @@ -0,0 +1,33 @@ +cabal-version: 2.4 +-- Initial package description 'external-package-user.cabal' generated by +-- 'cabal init'. For further documentation, see +-- http://haskell.org/cabal/users-guide/ + +name: external-package-user +version: 0.1.0.0 +synopsis: User package +description: A user of external-package-demo +-- bug-reports: +license: BSD-3-Clause +author: Maxim Koltsov +maintainer: kolmax94@gmail.com +-- copyright: +-- category: +extra-source-files: CHANGELOG.md + +library + exposed-modules: MyLib + -- other-modules: + -- other-extensions: + build-depends: base < 5, + external-package-demo + -- hs-source-dirs: + default-language: Haskell2010 + +executable external-package-user + main-is: Main.hs + other-modules: MyLib + -- other-extensions: + build-depends: base, external-package-user + -- hs-source-dirs: + default-language: Haskell2010 diff --git a/test/extra-hackage/hackage/default.nix b/test/extra-hackage/hackage/default.nix new file mode 100644 index 0000000000..483a407783 --- /dev/null +++ b/test/extra-hackage/hackage/default.nix @@ -0,0 +1,9 @@ +with builtins; mapAttrs (_: mapAttrs (_: data: rec { + inherit (data) sha256; + revisions = (mapAttrs (rev: rdata: { + inherit (rdata) revNum sha256; + outPath = ./. + "/hackage/${rdata.outPath}"; + }) data.revisions) // { + default = revisions."${data.revisions.default}"; + }; +})) (fromJSON (readFile ./hackage.json)) diff --git a/test/extra-hackage/hackage/hackage.json b/test/extra-hackage/hackage/hackage.json new file mode 100644 index 0000000000..5f1d652742 --- /dev/null +++ b/test/extra-hackage/hackage/hackage.json @@ -0,0 +1,15 @@ +{ + "external-package-demo": { + "0.1.0.0": { + "revisions": { + "default": "r0", + "r0": { + "outPath": "external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix", + "revNum": 0, + "sha256": "3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7" + } + }, + "sha256": "1fce0685dcb89200ed286b9ae0983322ebc8a2d5de0541da55a5b82797dba740" + } + } +} \ No newline at end of file diff --git a/test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix b/test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix new file mode 100644 index 0000000000..bf46e066c3 --- /dev/null +++ b/test/extra-hackage/hackage/hackage/external-package-demo-0.1.0.0-r0-3230db0813f2b468afb3ff7d8bbfcf570019a7faa4a0f59b2ea96743932105e7.nix @@ -0,0 +1,76 @@ +let + buildDepError = pkg: + builtins.throw '' + The Haskell package set does not contain the package: ${pkg} (build dependency). + + If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix. + ''; + sysDepError = pkg: + builtins.throw '' + The Nixpkgs package set does not contain the package: ${pkg} (system dependency). + + You may need to augment the system package mapping in haskell.nix so that it can be found. + ''; + pkgConfDepError = pkg: + builtins.throw '' + The pkg-conf packages does not contain the package: ${pkg} (pkg-conf dependency). + + You may need to augment the pkg-conf package mapping in haskell.nix so that it can be found. + ''; + exeDepError = pkg: + builtins.throw '' + The local executable components do not include the component: ${pkg} (executable dependency). + ''; + legacyExeDepError = pkg: + builtins.throw '' + The Haskell package set does not contain the package: ${pkg} (executable dependency). + + If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix. + ''; + buildToolDepError = pkg: + builtins.throw '' + Neither the Haskell package set or the Nixpkgs package set contain the package: ${pkg} (build tool dependency). + + If this is a system dependency: + You may need to augment the system package mapping in haskell.nix so that it can be found. + + If this is a Haskell dependency: + If you are using Stackage, make sure that you are using a snapshot that contains the package. Otherwise you may need to update the Hackage snapshot you are using, usually by updating haskell.nix. + ''; +in { system, compiler, flags, pkgs, hsPkgs, pkgconfPkgs, config, ... }: + { + flags = {}; + package = { + specVersion = "2.4"; + identifier = { name = "external-package-demo"; version = "0.1.0.0"; }; + license = "BSD-3-Clause"; + copyright = ""; + maintainer = "kolmax94@gmail.com"; + author = "Maxim Koltsov"; + homepage = ""; + url = ""; + synopsis = "Demo package."; + description = "Just a demo package..."; + buildType = "Simple"; + }; + components = { + "library" = { + depends = [ (hsPkgs."base" or (buildDepError "base")) ]; + buildable = true; + }; + exes = { + "external-package-demo" = { + depends = [ + (hsPkgs."base" or (buildDepError "base")) + (hsPkgs."external-package-demo" or (buildDepError "external-package-demo")) + ]; + buildable = true; + }; + }; + }; + } // { + src = (pkgs.lib).mkDefault (pkgs.fetchurl { + url = "http://not-there//package/external-package-demo-0.1.0.0/external-package-demo-0.1.0.0.tar.gz"; + sha256 = config.sha256; + }); + } \ No newline at end of file diff --git a/test/tests.sh b/test/tests.sh index 62f19024bb..3e57d1e5bf 100755 --- a/test/tests.sh +++ b/test/tests.sh @@ -104,4 +104,16 @@ printf "*** Checking the maintainer scripts...\n" >& 2 nix build $NIX_BUILD_ARGS --no-link --keep-going -f ../build.nix maintainer-scripts echo >& 2 +printf "*** Checking that plan construction works with extra Hackages...\n" >& 2 +nix build $NIX_BUILD_ARGS --no-link \ + -f /default.nix \ + extra-hackage.plan-nix +echo >& 2 + +printf "*** Checking that package with extra Hackages can be build...\n" >& 2 +nix build $NIX_BUILD_ARGS --no-link \ + -f /default.nix \ + extra-hackage.hsPkgs.external-package-user.components.all +echo >& 2 + printf "\n*** Finished successfully\n" >& 2