Error: Library "dune.configurator" not found. when using pkgsStatic

[Strum355]

Note: I am by no means an ocaml person, I know 0 ocaml, just trying to statically package comby 😄

Im getting build failures of the above kind when trying to build the following derivation using nix-ocaml/nix-overlays?rev=9287d78084ac4c0b69bff81cd6afacc4ff98c4a9. Apply the following patch to the above revision of sourcegraph/sourcegraph for it to be using nix-ocaml/nix-overlays:

git diff patch

diff --git a/flake.lock b/flake.lock
index 03634ff4d4..3c70103cdf 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,5 +1,20 @@
 {
   "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1678901627,
+        "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
+        "type": "github"
+      },
+      }
+    },
+    "ocaml": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1680538699,
+        "narHash": "sha256-0un9Iyt437edWOeZ3oP9iYhjYtCfQKCMprL902Eo2Ck=",
+        "owner": "nix-ocaml",
+        "repo": "nix-overlays",
+        "rev": "9287d78084ac4c0b69bff81cd6afacc4ff98c4a9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-ocaml",
+        "repo": "nix-overlays",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "nixpkgs": "nixpkgs",
+        "ocaml": "ocaml",
         "utils": "utils"
       }
     },
diff --git a/flake.nix b/flake.nix
index 2159574e29..333730ba79 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,9 +4,10 @@
   inputs = {
     nixpkgs.url = "nixpkgs/nixos-unstable";
     utils.url = "github:numtide/flake-utils";
+    ocaml.url = "github:nix-ocaml/nix-overlays";
   };
 
-  outputs = { self, nixpkgs, utils }:
+  outputs = { self, nixpkgs, utils, ocaml }:
     with nixpkgs.lib; with utils.lib; {
       devShells = eachDefaultSystem (system:
         let
@@ -23,7 +24,7 @@
       packages = fold recursiveUpdate { } [
         ((import ./dev/nix/ctags.nix { inherit nixpkgs utils; }).packages)
         (import ./dev/nix/p4-fusion.nix { inherit nixpkgs utils; })
-        (import ./dev/nix/comby.nix { inherit nixpkgs utils; })
+        (import ./dev/nix/comby.nix { inherit utils; nixpkgs = ocaml; })
       ];
     };
 }

full error output

@nix { "action": "setPhase", "phase": "unpackPhase" }
unpacking sources
unpacking source archive /nix/store/9hxcsw76gq991d6abh6nmjsm7rl1dd6k-source
source root is source
@nix { "action": "setPhase", "phase": "patchPhase" }
patching sources
@nix { "action": "setPhase", "phase": "updateAutotoolsGnuConfigScriptsPhase" }
updateAutotoolsGnuConfigScriptsPhase
@nix { "action": "setPhase", "phase": "configurePhase" }
configuring
no configure script, doing nothing
@nix { "action": "setPhase", "phase": "buildPhase" }
building
Scanned 0 directories
                     
Done: 0% (0/0, 0 left) (jobs: 0)
                                
...
 @nix { "action": "setPhase", "phase": "unpackPhase" }
unpacking sources
unpacking source archive /nix/store/9hxcsw76gq991d6abh6nmjsm7rl1dd6k-source
source root is source
@nix { "action": "setPhase", "phase": "patchPhase" }
patching sources
@nix { "action": "setPhase", "phase": "updateAutotoolsGnuConfigScriptsPhase" }
updateAutotoolsGnuConfigScriptsPhase
@nix { "action": "setPhase", "phase": "configurePhase" }
configuring
no configure script, doing nothing
@nix { "action": "setPhase", "phase": "buildPhase" }
building
Scanned 0 directories
                     
Done: 0% (0/0, 0 left) (jobs: 0)
                                
...
                                       
File "src/unix/config/dune", line 4, characters 12-29:
4 |  (libraries dune.configurator))
                ^^^^^^^^^^^^^^^^^
Error: Library "dune.configurator" not found.
-> required by _build/default/src/unix/config/discover.exe
-> required by _build/default/src/unix/lwt_features.h
-> required by _build/install/default/lib/lwt/unix/lwt_features.h
-> required by _build/default/lwt.install
-> required by alias install
Done: 25% (116/448, 332 left) (jobs: 0)
                                       
...
                                                 
(cd _build/default && /nix/store/3s8bqs0wbrc1wmvywqknm9p9725nx24x-ocaml+flambda-static-x86_64-unknown-linux-musl-4.14.1/bin/ocamlc.opt -w -40 -w +A-29 -g -bin-annot -I src/core/.lwt.objs/byte -intf-suffix .ml -no-alias-deps -o src/core/.lwt.objs/byte/lwt_stream.cmo -c -impl src/core/lwt_stream.ml)
File "src/core/lwt_stream.ml", line 46, characters 2-55:
46 |   mutable push_external : Obj.t [@ocaml.warning "-69"];
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 69 [unused-field]: record field push_external is never read.
(However, this field is used to build or mutate values.)
File "src/core/lwt_stream.ml", line 67, characters 2-56:
67 |   mutable pushb_external : Obj.t [@ocaml.warning "-69"];
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 69 [unused-field]: record field pushb_external is never read.
(However, this field is used to build or mutate values.)
Done: 71% (591/821, 230 left, 1 failed) (jobs: 6)
                                                 
...
                                                 
(cd _build/default && /nix/store/3s8bqs0wbrc1wmvywqknm9p9725nx24x-ocaml+flambda-static-x86_64-unknown-linux-musl-4.14.1/bin/ocamlopt.opt -w -40 -w +A-29 -g -I src/core/.lwt.objs/byte -I src/core/.lwt.objs/native -intf-suffix .ml -no-alias-deps -o src/core/.lwt.objs/native/lwt_stream.cmx -c -impl src/core/lwt_stream.ml)
File "src/core/lwt_stream.ml", line 46, characters 2-55:
46 |   mutable push_external : Obj.t [@ocaml.warning "-69"];
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 69 [unused-field]: record field push_external is never read.
(However, this field is used to build or mutate values.)
File "src/core/lwt_stream.ml", line 67, characters 2-56:
67 |   mutable pushb_external : Obj.t [@ocaml.warning "-69"];
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Warning 69 [unused-field]: record field pushb_external is never read.
(However, this field is used to build or mutate values.)
Done: 76% (627/821, 194 left, 1 failed) (jobs: 1)
                                                 
Done: 76% (629/821, 192 left, 1 failed) (jobs: 1)
                                                 
Saving build trace db...
                        
Saving digest db...

Any pointers or solutions much appreciated!

[anmonteiro]

I think you can simplify the setup a bit when using these nix-overlays. Here's an example for building comby-musl: anmonteiro/sourcegraph@245bb5b

(The pkg-config hack should go away in the next version of dune, cf ocaml/dune#7469)

[Strum355]

Thanks for the fast reply! I tried out your fork but I dont seem to be able to get it to emit a static binary:

$ nix build .#comby-musl
warning: Git tree '/tmp/sourcegraph' is dirty
$ ldd ./result/bin/comby
	linux-vdso.so.1 (0x00007ffdf37fe000)
	libsqlite3.so.0 => /nix/store/5yj9mspralsf1qawaa3pmr14cppsrzmm-sqlite-x86_64-unknown-linux-musl-3.41.1/lib/libsqlite3.so.0 (0x00007f786e6b0000)
	libz.so.1 => /nix/store/hvd08d266j17651m741d10z7js7vgpb5-zlib-x86_64-unknown-linux-musl-1.2.13/lib/libz.so.1 (0x00007f78717c2000)
	libev.so.4 => /nix/store/25bc6g2xdddi0gi0r8d4501x8f3c61mi-libev-x86_64-unknown-linux-musl-4.33/lib/libev.so.4 (0x00007f78717b0000)
	libpcre.so.1 => /nix/store/982spfnh8ss82nga4g7f5ihb80cr0x73-pcre-x86_64-unknown-linux-musl-8.45/lib/libpcre.so.1 (0x00007f7871736000)
	libc.so => /nix/store/c821rry6lp02mfv0njsfc3r76zygzzzd-musl-x86_64-unknown-linux-musl-1.2.3/lib/libc.so (0x00007f786e5fb000)

even with the following patch

diff --git a/dev/nix/comby.nix b/dev/nix/comby.nix
index 6f92cee..95f52ed 100644
--- a/dev/nix/comby.nix
+++ b/dev/nix/comby.nix
@@ -31,7 +31,7 @@ nixpkgs.lib.genAttrs utils.lib.defaultSystems (system:
     {
       comby-musl =
         let
-          inherit (pkgs) stdenv writeScriptBin pkg-config;
+          inherit (pkgs.pkgsStatic) stdenv writeScriptBin pkg-config;
           pkg-config-script =
             let
               pkg-config-pkg =
@@ -44,7 +44,7 @@ nixpkgs.lib.genAttrs utils.lib.defaultSystems (system:
               ${pkg-config-pkg} $@
             '';
         in
-        pkgs.pkgsCross.musl64.comby.overrideAttrs (o: {
+        pkgs.pkgsStatic.pkgsCross.musl64.comby.overrideAttrs (o: {
           nativeBuildInputs = o.nativeBuildInputs ++ [ pkg-config-script ];
         });

I was able to get as far as the below output with the existing nix after patchelf (with a minor change, first param to combyBuilder being pkgs.pkgsMusl instead of pkgs.pkgsStatic):

$ ldd comby 
        /lib/ld-musl-x86_64.so.1 (0x7fe5466d2000)
        libc.so => /lib/ld-musl-x86_64.so.1 (0x7fe5466d2000)

[anmonteiro]

That seemed too easy 😅 I think I can make this work but it's gonna take a little bit longer. Standby.

[anmonteiro]

Alright, I pushed a new commit to that branch (https://github.com/sourcegraph/sourcegraph/commit/ab5d0f8ff3bfbdb2b08d9cdd1d29d6ddc4ca5581).

It builds a statically linked executable now:

$ nix run github:nix-ocaml/nix-overlays#file -- ./result/bin/comby
./result/bin/comby: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, not stripped

While we don't need the pkg-config trick anymore, there was something I couldn't make work with the vendored camlzip, so that's mostly what the included patches do.

It needs #733 which should finish the CI run shortly.

[anmonteiro]

@Strum355 PR closed this automatically, but feel free to reopen if my fix didn't work.