WIP: refactor go builder

pkgs/build-support/go/build-hook.sh

@@ -0,0 +1,139 @@
+
+goBuildHook() {
+
+  runHook preBuild
+
+  export GOFLAGS
+
+  # currently pie is only enabled by default in pkgsMusl
+  # this will respect the `hardening{Disable,Enable}` flags if set
+  if [[ $NIX_HARDENING_ENABLE =~ "pie" ]]; then
+    prependToVar GOFLAGS "-buildmode=pie"
+  fi
+
+  if [ -z "${allowGoReference-}" ]; then
+    appendToVar GOFLAGS "-trimpath"
+  fi
+
+  if [ -z "${proxyVendor-}" ]; then
+    appendToVar GOFLAGS "-mod=vendor"
+  fi
+
+  # TODO: conditionalize
+  appendToVar ldflags "-buildid="
+
+  exclude='\(/_\|examples\|Godeps\|testdata'
+  if [[ -n "$excludedPackages" ]]; then
+    IFS=' ' read -r -a excludedArr <<<$excludedPackages
+    printf -v excludedAlternates '%s\\|' "${excludedArr[@]}"
+    excludedAlternates=${excludedAlternates%\\|} # drop final \| added by printf
+    exclude+='\|'"$excludedAlternates"
+  fi
+  exclude+='\)'
+
+  buildGoDir() {
+    local cmd="$1" dir="$2"
+
+    declare -ga buildFlagsArray
+    declare -a flags
+    flags+=($buildFlags "${buildFlagsArray[@]}")
+    flags+=(${tags:+-tags=${tags// /,}})
+    flags+=(${ldflags:+-ldflags="$ldflags"})
+    flags+=("-p" "$NIX_BUILD_CORES")
+
+    if [ "$cmd" = "test" ]; then
+      flags+=(-vet=off)
+      flags+=($checkFlags)
+    fi
+
+    local OUT
+    if ! OUT="$(go $cmd "${flags[@]}" $dir 2>&1)"; then
+      if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+        echo "$OUT" >&2
+        return 1
+      fi
+    fi
+    if [ -n "$OUT" ]; then
+      echo "$OUT" >&2
+    fi
+    return 0
+  }
+
+  getGoDirs() {
+    local type;
+    type="$1"
+    if [ -n "$subPackages" ]; then
+      echo "$subPackages" | sed "s,\(^\| \),\1./,g"
+    else
+      find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude"
+    fi
+  }
+
+  if (( "${NIX_DEBUG:-0}" >= 1 )); then
+    buildFlagsArray+=(-x)
+  fi
+
+  if [ -z "$enableParallelBuilding" ]; then
+      export NIX_BUILD_CORES=1
+  fi
+
+  if [ -n "${modRoot}" ]; then
+    pushd "$modRoot"
+  fi
+
+  for pkg in $(getGoDirs ""); do
+    echo "Building subPackage $pkg"
+    buildGoDir install "$pkg"
+  done
+
+  if [ -n "${modRoot}" ]; then
+    popd
+  fi
+
+  # TODO: maybe conditionalize
+  # normalize cross-compiled builds w.r.t. native builds
+  (
+    dir="$GOPATH"/bin/"$GOOS"_"$GOARCH"
+    if [[ -n "$(shopt -s nullglob; echo "$dir"/*)" ]]; then
+      mv "$dir"/* "$dir"/..
+    fi
+    if [[ -d "$dir" ]]; then
+      rmdir "$dir"
+    fi
+  )
+
+  runHook postBuild
+}
+
+goCheckHook() {
+  runHook preCheck
+
+  # We do not set trimpath for tests, in case they reference test assets
+  export GOFLAGS="${GOFLAGS//-trimpath/}"
+
+  if [ -n "${modRoot}" ]; then
+    pushd "$modRoot"
+  fi
+
+  for pkg in $(getGoDirs test); do
+    buildGoDir test "$pkg"
+  done
+
+  if [ -n "${modRoot}" ]; then
+    popd
+  fi
+
+  runHook postCheck
+}
+
+
+if [ -z "${dontGoBuild-}" ] && [ -z "${buildPhase-}" ]; then
+  buildPhase=goBuildHook
+fi
+
+
+if [ -z "${dontGoCheck-}" ] && [ -z "${checkPhase-}" ]; then
+  checkPhase=goCheckHook
+fi
+
+

pkgs/build-support/go/config-hook.sh

@@ -0,0 +1,28 @@
+goConfigHook() {
+  export GOCACHE=$TMPDIR/go-cache
+  export GOPATH="$TMPDIR/go"
+  export GOPROXY=off
+  export GOSUMDB=off
+
+  if [ -d "${goModules-}" ]; then
+    if [ -n "${proxyVendor-}" ]; then
+      export GOPROXY="file://$goModules"
+    else
+
+      if [ -n "${modRoot}" ]; then
+        pushd "$modRoot"
+      fi
+
+      rm -rf vendor
+      cp -r --reflink=auto "$goModules" vendor
+
+      if [ -n "${modRoot}" ]; then
+        popd
+      fi
+    fi
+  else
+    echo "goModules is not set or incorrect"
+  fi
+}
+
+postConfigureHooks+=(goConfigHook)

pkgs/build-support/go/install-hook.sh

@@ -0,0 +1,14 @@
+goInstallHook() {
+  runHook preInstall
+
+  mkdir -p "$out"
+  dir="$GOPATH/bin"
+  [ -e "$dir" ] && cp -r "$dir" "$out"
+
+  runHook postInstall
+}
+
+
+if [ -z "${dontGoInstall-}" ] && [ -z "${installPhase-}" ]; then
+    installPhase=goInstallHook
+fi

pkgs/build-support/go/module2.nix

@@ -0,0 +1,126 @@
+{
+  fetchGoModVendor,
+  makeSetupHook,
+  go,
+  cacert,
+  git,
+  lib,
+  stdenv,
+}:
+
+args:
+
+let
+  # these are already handled manually
+  filteredArgs = removeAttrs args [
+    "overrideModAttrs"
+    "nativeBuildInputs"
+    "passthru"
+    "meta"
+  ];
+
+  configHook = makeSetupHook {
+    name = "go-config-hook";
+  } ./config-hook.sh;
+
+  buildHook = makeSetupHook {
+    name = "go-build-hook";
+  } ./build-hook.sh;
+
+  installHook = makeSetupHook {
+    name = "go-install-hook";
+  } ./install-hook.sh;
+
+in
+
+(stdenv.mkDerivation (
+  finalAttrs:
+  {
+
+    # The SRI hash of the vendored dependencies.
+    # If `vendorHash` is `null`, no dependencies are fetched and
+    # the build relies on the vendor folder within the source.
+    vendorHash = throw "buildGoModule: vendorHash is missing";
+
+    # Directory to the `go.mod` and `go.sum` relative to the `src`.
+    modRoot = "./";
+
+    # Whether to delete the vendor folder supplied with the source.
+    deleteVendor = false;
+
+    # Whether to fetch (go mod download) and proxy the vendor directory.
+    # This is useful if your code depends on c code and go mod tidy does not
+    # include the needed sources to build or if any dependency has case-insensitive
+    # conflicts which will produce platform dependant `vendorHash` checksums.
+    proxyVendor = false;
+
+    # Do not enable this without good reason
+    # IE: programs coupled with the compiler.
+    allowGoReference = false;
+
+    goModules =
+      if (finalAttrs.vendorHash == null) then
+        null
+      else
+        (fetchGoModVendor {
+          inherit (finalAttrs) pname version;
+          inherit (finalAttrs) src; # srcs, patches, etc
+          inherit (finalAttrs)
+            modRoot
+            deleteVendor
+            proxyVendor
+            enableParallelBuilding
+            ;
+          hash = finalAttrs.vendorHash;
+        }).overrideAttrs
+          finalAttrs.passthru.overrideModAttrs;
+
+    # We want parallel builds by default.
+    enableParallelBuilding = true;
+
+    strictDeps = true;
+
+    nativeBuildInputs = [
+      go
+      configHook
+      buildHook
+      installHook
+    ] ++ args.nativeBuildInputs or [ ];
+
+    # TODO: should probably be moved to the hooks
+    inherit (go) GOOS GOARCH CGO_ENABLED;
+
+    # TODO: what is this?
+    GO111MODULE = "on";
+    GOTOOLCHAIN = "local";
+
+    doCheck = true;
+
+    disallowedReferences = lib.optional (!finalAttrs.allowGoReference) go;
+
+    passthru = {
+      inherit go;
+      overrideModAttrs = args.overrideModAttrs or (finalAttrs: previousAttrs: { });
+    } // args.passthru or { };
+
+    meta = {
+      # Add default meta information.
+      platforms = go.meta.platforms or lib.platforms.all;
+    } // args.meta or { };
+
+    # TODO: remove or move to bash
+    __bruh =
+      lib.warnIf (lib.any (lib.hasPrefix "-mod=") (args.GOFLAGS or [ ]))
+        "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS"
+        (
+          lib.warnIf (builtins.elem "-trimpath" args.GOFLAGS or [ ])
+            "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true"
+            (
+              lib.warnIf (builtins.elem "-buildid="
+                args.ldflags or [ ]
+              ) "`-buildid=` is set by default as ldflag by buildGoModule" null
+            )
+        );
+  }
+  // filteredArgs
+))