From 458a058138e823ef388d96382a02d575b9c0ae87 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 19 Oct 2025 20:40:42 -0400 Subject: [PATCH 1/2] Add prebuilt-depends for proprietary dependencies. --- builder/comp-builder.nix | 8 ++++++-- builder/hspkg-builder.nix | 1 + builder/make-config-files.nix | 27 ++++++++++++++------------- lib/call-cabal-project-to-nix.nix | 11 +++++++++++ modules/cabal-project.nix | 5 +++++ modules/component-driver.nix | 5 +++++ overlays/haskell.nix | 1 + 7 files changed, 43 insertions(+), 15 deletions(-) diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix index 819cfea2ab..69ce9df659 100644 --- a/builder/comp-builder.nix +++ b/builder/comp-builder.nix @@ -90,6 +90,8 @@ # LLVM , useLLVM ? ghc.useLLVM or false , smallAddressSpace ? false + +, prebuilt-depends ? [] }: # makeOverridable is called here after all the `? DEFAULT` arguments # will have been applied. This makes sure that `c.override (oldAttrs: {...})` @@ -160,6 +162,7 @@ let self = , enableTSanRTS , useLLVM , smallAddressSpace +, prebuilt-depends }@drvArgs: let @@ -213,7 +216,7 @@ let configFiles = makeConfigFiles { component = componentForSetup; inherit (package) identifier; - inherit fullName flags needsProfiling enableDWARF; + inherit fullName flags needsProfiling enableDWARF prebuilt-depends; }; enableFeature = enable: feature: @@ -852,5 +855,6 @@ in drv; in self) { enableDWARF enableTSanRTS useLLVM - smallAddressSpace; + smallAddressSpace + prebuilt-depends; } diff --git a/builder/hspkg-builder.nix b/builder/hspkg-builder.nix index 7a24ebea2f..373eacf666 100644 --- a/builder/hspkg-builder.nix +++ b/builder/hspkg-builder.nix @@ -58,6 +58,7 @@ let inherit allComponent componentId component package name src flags setup cabalFile cabal-generator patches shellHook ; + inherit (config) prebuilt-depends; }; in rec { diff --git a/builder/make-config-files.nix b/builder/make-config-files.nix index 2aec4fe63d..e4c0c89f08 100644 --- a/builder/make-config-files.nix +++ b/builder/make-config-files.nix @@ -1,6 +1,6 @@ { stdenv, lib, haskellLib, ghc, nonReinstallablePkgs, runCommand, writeText, writeScript }@defaults: -{ identifier, component, fullName, flags ? {}, needsProfiling ? false, enableDWARF ? false, chooseDrv ? drv: drv, nonReinstallablePkgs ? defaults.nonReinstallablePkgs }: +{ identifier, component, fullName, flags ? {}, needsProfiling ? false, enableDWARF ? false, chooseDrv ? drv: drv, nonReinstallablePkgs ? defaults.nonReinstallablePkgs, prebuilt-depends ? [] }: let # Sort and remove duplicates from nonReinstallablePkgs. @@ -57,7 +57,7 @@ let ((if needsProfiling then (x: map (p: p.profiled or p) x) else x: x) (map haskellLib.dependToLib component.depends)) ) - ); + ) ++ prebuilt-depends; script = '' ${target-pkg} init $configFiles/${packageCfgDir} @@ -149,17 +149,6 @@ let echo "allow-older: ${identifier.name}:*" >> $configFiles/cabal.config ''} - for p in "''${pkgsHostTarget[@]}"; do - if [ -e $p/envDep ]; then - cat $p/envDep >> $configFiles/ghc-environment - fi - ${ lib.optionalString component.doExactConfig '' - if [ -d $p/exactDep ]; then - cat $p/exactDep/configure-flags >> $configFiles/configure-flags - cat $p/exactDep/cabal.config >> $configFiles/cabal.config - fi - ''} - done for p in ${lib.concatStringsSep " " nonReinstallablePkgs'}; do if [ -e $ghcDeps/envDeps/$p ]; then cat $ghcDeps/envDeps/$p >> $configFiles/ghc-environment @@ -172,6 +161,18 @@ let cat $ghcDeps/exactDeps/$p/cabal.config >> $configFiles/cabal.config fi done + '' + '' + for p in "''${pkgsHostTarget[@]}"; do + if [ -e $p/envDep ]; then + cat $p/envDep >> $configFiles/ghc-environment + fi + ${ lib.optionalString component.doExactConfig '' + if [ -d $p/exactDep ]; then + cat $p/exactDep/configure-flags >> $configFiles/configure-flags + cat $p/exactDep/cabal.config >> $configFiles/cabal.config + fi + ''} + done '' # This code originates in the `generic-builder.nix` from nixpkgs. However GHC has been fixed # to drop unused libraries referenced from libraries; and this patch is usually included in the diff --git a/lib/call-cabal-project-to-nix.nix b/lib/call-cabal-project-to-nix.nix index 34dec9b988..520d775bf0 100644 --- a/lib/call-cabal-project-to-nix.nix +++ b/lib/call-cabal-project-to-nix.nix @@ -62,6 +62,7 @@ , evalPackages , supportHpack ? false # Run hpack on package.yaml files with no .cabal file , ignorePackageYaml ? false # Ignore package.yaml files even if they exist +, prebuilt-depends ? [] , ... }@args: let @@ -371,6 +372,7 @@ let }; dummy-ghc-pkg-dump = evalPackages.runCommand "dummy-ghc-pkg-dump" { + buildInputs = prebuilt-depends; nativeBuildInputs = [ evalPackages.haskell-nix.nix-tools-unchecked.exes.cabal2json evalPackages.jq @@ -535,6 +537,15 @@ let LAST_PKG="ghcjs-th" '' } + for l in "''${pkgsHostTarget[@]}"; do + if [ -d "$l/package.conf.d" ]; then + files=("$l/package.conf.d/"*.conf) + for file in "''${files[@]}"; do + cat "$file" >> $out + echo '---' >> $out + done + fi + done for pkg in $PKGS; do varname="$(echo $pkg | tr "-" "_")" ver="VER_$varname" diff --git a/modules/cabal-project.nix b/modules/cabal-project.nix index 0b7316c84d..60de09fae5 100644 --- a/modules/cabal-project.nix +++ b/modules/cabal-project.nix @@ -144,5 +144,10 @@ in { type = nullOr (listOf unspecified); default = []; }; + prebuilt-depends = mkOption { + type = listOf package; + default = []; + description = "pre-built (perhaps proprietary) Haskell packages to make available as dependencies"; + }; }; } diff --git a/modules/component-driver.nix b/modules/component-driver.nix index 118db7c55d..318f9f35d1 100644 --- a/modules/component-driver.nix +++ b/modules/component-driver.nix @@ -37,6 +37,11 @@ in default = []; description = "pkgs to globally provide to Setup.hs builds"; }; + options.prebuilt-depends = lib.mkOption { + type = lib.types.listOf lib.types.package; + default = []; + description = "pre-built (perhaps proprietary) Haskell packages to make available as dependencies"; + }; # Dependencies (with reinstallable-lib:ghc) # diff --git a/overlays/haskell.nix b/overlays/haskell.nix index c9084f2e6f..89dc911cbd 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -699,6 +699,7 @@ final: prev: { in if ghc.isHaskellNixCompiler or false then ghc.override { ghcEvalPackages = evalPackages; } else ghc; compiler.nix-name = final.lib.mkForce config.compiler-nix-name; evalPackages = final.lib.mkDefault evalPackages; + inherit (config) prebuilt-depends; } ]; extra-hackages = config.extra-hackages or [] ++ callProjectResults.extra-hackages; }; From 5ab033ebad847b0ecbbbf6ae9a1b96a5f91bcba3 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 20 Oct 2025 06:53:17 -0400 Subject: [PATCH 2/2] fixup! Add prebuilt-depends for proprietary dependencies. Add commentary --- builder/comp-builder.nix | 57 +++++++++++++++++++++++++++++++++++ builder/make-config-files.nix | 7 ++++- modules/cabal-project.nix | 6 +++- modules/component-driver.nix | 6 +++- 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/builder/comp-builder.nix b/builder/comp-builder.nix index 69ce9df659..be06ce38d1 100644 --- a/builder/comp-builder.nix +++ b/builder/comp-builder.nix @@ -91,6 +91,63 @@ , useLLVM ? ghc.useLLVM or false , smallAddressSpace ? false +# Note [prebuilt dependencies] +# +# Typical cabal project planning starts with the libraries that come with +# the compiler and then plans to build every other needed dependency from +# source (fetched through hackage repositories or source-repository +# dependencies). In cases where some library that isn't part of the compiler +# is only available as a pre-built shared object file (such as for some +# closed-source module from a vendor, or in principle a component whose +# compilation is extremely expensive), we need to be able to tell cabal +# about additional prebuilt dependencies to include in its plan and link to +# as needed at build time. +# +# This can be done by passing the needed libraries in prebuilt-depends. During +# cabal planning and builds, these libraries (and their dependencies) will be +# present in the ghc-pkg database that cabal will draw from for its dependency +# resolution, thereby skipping lookup from hackage or building from source. +# +# The entries in the prebuilt dependencies list may have dependencies that are +# part of the compiler-provided package set, or may have overlap with each other. +# GHC can actually handle this use case fine, since types from different packages +# (even of the same name) will not unify, so at worst you will get a compile error, +# but cabal will need to choose one for the packages you are building. The entries +# in the list are given priority over the compiler-provided ones, with the later +# entries having greater priority than the earlier ones. +# +# For your build to succeed, your prebuilt-dependencies must meet the following: +# +# 1. They are built with the same compiler version and RTS way. +# 2. They have all of their dependencies within the package db directory or +# included as propagatedBuildInputs +# 3. They must include the `envDep` and `exactDep` files that make-config.files.nix +# expects for configuring cabal precisely. +# +# The recommended way to meet this requirement is to build the relevant libraries +# with haskell.nix too, since it sets up the dependencies appropriately. An example +# workflow would be: +# +# 1. Build libraries foo and bar with haskell.nix (the same plan) +# 2. Note down the store paths for foo and bar library outputs +# 3. Make a full nix export of those store paths (using e.g. `nix-store --export $(nix-store --query --requisites $barPath $fooPath) > foobar.closure` +# 4. On the consumer machine, import the store paths (e.g. `nix-store --import foobar.closure` and then add gc roots) +# 5. In the consumer haskell.nix build, add the imported store paths to your prebuilt-depends. E.g.: +# +# prebuilt-depends = let foo = { +# # We need to make this look like a call to derivation for stdenv to do the right thing +# name = "foo-lib-foo-0.1.0.0"; +# type = "derivation"; +# outputs = [ "out" ]; +# out = foo; +# all = [ foo ]; +# # $fooPath is already in the store due to the import, so we use the storePath +# # builtin to add it as a source dependency to the build. Note that this does +# # not work in pure evaluation mode, you must use --impure with flakes. An +# # alternative would be to bundle up all of the needed libraries into tarballs that +# # are fetched and unpacked as proper fixed-output derivations. +# outPath = builtins.storePath $fooPath; +# }; in [ foo ]; # Could also do the same for bar of course , prebuilt-depends ? [] }: # makeOverridable is called here after all the `? DEFAULT` arguments diff --git a/builder/make-config-files.nix b/builder/make-config-files.nix index e4c0c89f08..bdf5673939 100644 --- a/builder/make-config-files.nix +++ b/builder/make-config-files.nix @@ -161,7 +161,12 @@ let cat $ghcDeps/exactDeps/$p/cabal.config >> $configFiles/cabal.config fi done - '' + '' + '' + # We put the packages in buildInputs *after* the GHC deps on the command line + # to ensure that any prebuilt dependencies (see Note [prebuilt dependencies]) + # take priority over the GHC-provided ones. We can't relink the prebuilt + # libraries, so this is the most likely to avoid conflicts. + + '' for p in "''${pkgsHostTarget[@]}"; do if [ -e $p/envDep ]; then cat $p/envDep >> $configFiles/ghc-environment diff --git a/modules/cabal-project.nix b/modules/cabal-project.nix index 60de09fae5..0f2e33025a 100644 --- a/modules/cabal-project.nix +++ b/modules/cabal-project.nix @@ -147,7 +147,11 @@ in { prebuilt-depends = mkOption { type = listOf package; default = []; - description = "pre-built (perhaps proprietary) Haskell packages to make available as dependencies"; + description = '' + pre-built (perhaps proprietary) Haskell packages to make available as dependencies + + See Note [prebuilt dependencies] for more details + ''; }; }; } diff --git a/modules/component-driver.nix b/modules/component-driver.nix index 318f9f35d1..abfefc91a5 100644 --- a/modules/component-driver.nix +++ b/modules/component-driver.nix @@ -40,7 +40,11 @@ in options.prebuilt-depends = lib.mkOption { type = lib.types.listOf lib.types.package; default = []; - description = "pre-built (perhaps proprietary) Haskell packages to make available as dependencies"; + description = '' + pre-built (perhaps proprietary) Haskell packages to make available as dependencies + + See Note [prebuilt dependencies] for more details + ''; }; # Dependencies (with reinstallable-lib:ghc)