nix: switch status-im config to env vars, cleanup

To help with #15595 changes, refactoring is required.
In this PR we switch from config to env vars.
Doing some cleanup meanwhile.

Makefile

@@ -46,7 +46,10 @@ export NIX_USER_CONF_FILES = $(PWD)/nix/nix.conf
 # Location of symlinks to derivations that should not be garbage collected
 export _NIX_GCROOTS = /nix/var/nix/gcroots/per-user/$(USER)/status-mobile
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
-export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE
+export _NIX_KEEP ?= TMPDIR,BUILD_ENV,\
+	BUILD_TYPE,BUILD_NUMBER,COMMIT_HASH,\
+	ANDROID_GRADLE_OPTS,ANDROID_ABI_SPLIT,ANDROID_ABI_INCLUDE,\
+	STATUS_GO_SRC_OVERRIDE,STATUS_GO_IPFS_GATEWAY_URL
 
 # Useful for Android release builds
 TMP_BUILD_NUMBER := $(shell ./scripts/version/gen_build_no.sh | cut -c1-10)

ci/Jenkinsfile.android

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.7.9'
+library 'status-jenkins-lib@nix_switch_config_to_env'
 
 /* Options section can't access functions in objects. */
 def isPRBuild = utils.isPRBuild()

nix/DETAILS.md

@@ -75,13 +75,6 @@ nix-build \
   --attr targets.mobile.android.release \
   --argstr secrets-file '/tmp/tmp-status-mobile-559a3a441/tmp.xAnrPuNtAP' \
   --option extra-sandbox-paths '/home/joe/.gradle/status-im.keystore /tmp/tmp-status-mobile-559a3a441/tmp.xAnrPuNtAP' \
-  --arg config '{ \
-    status-im.build-type="nightly";
-    status-im.build-number="2020022418";
-    status-im.android.keystore-path="/home/joe/.gradle/status-im.keystore";
-    status-im.android.abi-split="false";
-    status-im.android.abi-include="armeabi-v7a;arm64-v8a;x86";
-  }' \
   default.nix
 ```
 Some of those are required which is why just calling:

nix/README.md

@@ -8,27 +8,27 @@ The main config file is [`nix/nix.conf`](/nix/nix.conf) and its main purpose is
 
 ## Build arguments
 
-We leverage the `config` argument of standard `nixpkgs` for our own parameterization of the builds (e.g. to pass a build number or build type).
-
-Here is a sample structure of the `config` attribute set:
-
-```nix
-{
-  status-im = {
-    build-type = "pr";     # Build type (influences which .env file gets used for feature flags)
-    build-number = 9999;   # Used for versionCode and CFBundleVersion in Android and iOS respectively
-    android = {
-      gradle-opts = "";    # Gradle options passed for Android builds
-      abi-split = false;   # If APKs should be split based on architectures
-      abi-include = "x86"; # Android architectures to build for
-    };
-    status-go = {
-      src-override = "$HOME/my/source/status-go"; # local source override
-    };
-  };
-}
+We leverage the environment variables for parameterization of the builds (e.g. to pass a build number or build type).
+
+Here is the list of variables you can use to override defaults:
+```shell
+# Build type (influences which .env file gets used for feature flags)
+BUILD_TYPE = "pr"
+# Used for versionCode and CFBundleVersion in Android and iOS respectively
+BUILD_NUMBER = 9999
+# Gradle options passed for Android builds
+ANDROID_GRADLE_OPTS = ""
+# If APKs should be split based on architectures
+ANDROID_ABI_SPLIT = false
+# Android architectures to build for
+ANDROID_ABI_INCLUDE = "x86"
+# Local source override, see below
+STATUS_GO_SRC_OVERRIDE = "$HOME/my/source/status-go"
+STATUS_GO_IPFS_GATEWAY_URL = "https://ipfs.status.im/"
 ```
-You can see the defaults in [`nix/config.nix`](./config.nix).
+You can see the defaults in code:
+- [`nix/status-go/default.nix`](./status-go/default.nix)
+- [`nix/mobile/android/release.nix`](./mobile/android/release.nix)
 
 ## Shell
 

nix/config.nix

@@ -1,23 +1,5 @@
 # Status defaults for config
 {
-  status-im = {
-    build-type = "pr";       # Build type (influences which .env file gets used for feature flags)
-    build-number = 9999;     # Used for versionCode and CFBundleVersion in Android and iOS respectively
-    commit-hash = "unknown"; # Included in APK Manifest for easier identification.
-
-    android = {
-      gradle-opts = null;    # Gradle options passed for Android builds
-      apk-signed = true;     # F-Droid builds aren't signed by us
-      abi-split = false;     # If APKs should be split based on architectures
-      abi-include = "armeabi-v7a;arm64-v8a;x86"; # Android architectures to build for
-    };
-
-    status-go = {
-      src-override = null;
-      ipfs-gateway-url = "https://ipfs.status.im/";
-    };
-  };
-
   # Android SDK requires an accepted license
   android_sdk.accept_license = true;
   allowUnfree = true;

nix/lib/default.nix

@@ -1,8 +1,9 @@
-{ lib, config }:
+{ lib }:
 
 {
-  getConfig = import ./getConfig.nix { inherit lib config; };
+  getEnvWithDefault = import ./getEnvWithDefault.nix {};
   mkFilter = import ./mkFilter.nix { inherit lib; };
   mergeSh = import ./mergeSh.nix { inherit lib; };
   checkEnvVarSet = import ./checkEnvVarSet.nix;
+  sanitizeVersion = import ./sanitizeVersion.nix { inherit lib; };
 }

nix/lib/getEnvWithDefault.nix

@@ -0,0 +1,11 @@
+# helper for getting environment variables in a safe way
+#
+
+{ }:
+
+name: default:
+let
+  envValue = builtins.getEnv name;
+  returnValue = if envValue != "" then envValue else default;
+in
+  builtins.trace "getEnvWithDefault ${name} (default: ${toString default}): ${toString returnValue}" returnValue

nix/lib/sanitizeVersion.nix

@@ -0,0 +1,14 @@
+{ lib }:
+
+version:
+let
+  # paths don't like slashes in them
+  dropSlashes = builtins.replaceStrings [ "/" ] [ "_" ];
+  # if version doesn't match this it's probably a commit, it's lax semver
+  versionRegex = "^v?[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+[[:alnum:]+_.-]*$";
+in
+  if (builtins.match versionRegex version) != null
+  # Geth forces a 'v' prefix for all versions
+  then lib.removePrefix "v" (dropSlashes version)
+  # reduce metrics cardinality in Prometheus
+  else lib.traceValFn (_: "WARNING: Marking build version as 'develop'!") "develop"

nix/mobile/android/release.nix

@@ -6,20 +6,30 @@
   buildEnv ? "prod",
   # Path to the file containing secret environment variables
   secretsFile ? "",
+  # Build type (influences which .env file gets used for feature flags)
+  # TODO: pr or relase for default?
+  buildType ? lib.getEnvWithDefault "BUILD_TYPE" "release",
+  # Used for versionCode
+  buildNumber ? lib.getEnvWithDefault "BUILD_NUMBER" 9999,
+  # Included in APK Manifest for easier identification.
+  # TODO: or GIT_COMMIT from Jenkins?
+  commitHash ? lib.getEnvWithDefault "COMMIT_HASH" "unknown",
+  # Gradle options passed for Android builds
+  androidGradleOpts ? lib.getEnvWithDefault "ANDROID_GRADLE_OPTS" null,
+  statusGoSrcOverride ? lib.getEnvWithDefault "STATUS_GO_SRC_OVERRIDE" null,
+  # If APKs should be split based on architectures
+  androidAbiSplit ? lib.getEnvWithDefault "ANDROID_ABI_SPLIT" "false",
+  # Android architectures to build for
+  # Used to detect end-to-end builds
+  androidAbiInclude ? lib.getEnvWithDefault "ANDROID_ABI_INCLUDE" "armeabi-v7a;arm64-v8a;x86",
 }:
 
 let
-  inherit (lib) toLower optionalString stringLength getConfig makeLibraryPath elem;
+  inherit (lib) toLower optionalString stringLength makeLibraryPath elem;
 
   # Pass secretsFile for POKT_TOKEN to jsbundle build
   builtJsBundle = jsbundle { inherit secretsFile; };
 
-  buildType = getConfig "build-type" "release";
-  buildNumber = getConfig "build-number" 9999;
-  commitHash = getConfig "commit-hash" "unknown";
-  gradleOpts = getConfig "android.gradle-opts" null;
-  # Used to detect end-to-end builds
-  androidAbiInclude = getConfig "android.abi-include" "armeabi-v7a;arm64-v8a;x86";
 
   envFileName =
     if androidAbiInclude == "x86"                  then ".env.e2e"
@@ -63,9 +73,8 @@ in stdenv.mkDerivation rec {
   # Used by Clojure at compile time to include JS modules
   BUILD_ENV = buildEnv;
 
-  # custom env variables derived from config
-  STATUS_GO_SRC_OVERRIDE = getConfig "status-go.src-override" null;
-  ANDROID_ABI_SPLIT = getConfig "android.abi-split" "false";
+  STATUS_GO_SRC_OVERRIDE = statusGoSrcOverride;
+  ANDROID_ABI_SPLIT = androidAbiSplit;
   ANDROID_ABI_INCLUDE = androidAbiInclude;
 
   # Android SDK/NDK for use by Gradle
@@ -122,17 +131,9 @@ in stdenv.mkDerivation rec {
   buildPhase = let
     adhocEnvVars = optionalString stdenv.isLinux
       "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${makeLibraryPath [ pkgs.zlib ]}";
-  in
-    assert ANDROID_ABI_SPLIT != null && ANDROID_ABI_SPLIT != "";
-    assert stringLength ANDROID_ABI_INCLUDE > 0;
-  ''
-    # Fixes issue with failing to load libnative-platform.so
-    export GRADLE_USER_HOME=$(mktemp -d)
-    export ANDROID_SDK_HOME=$(mktemp -d)
-
-    pushd ./android
-    ${adhocEnvVars} ${pkgs.gradle}/bin/gradle \
-      ${toString gradleOpts} \
+    gradleCommand = ''
+      ${pkgs.gradle}/bin/gradle \
+      ${toString androidGradleOpts} \
       --console=plain \
       --offline \
       --no-daemon \
@@ -142,8 +143,21 @@ in stdenv.mkDerivation rec {
       -Dmaven.repo.local='${deps.gradle}' \
       -PversionCode=${toString buildNumber} \
       -PcommitHash=${commitHash} \
-      assemble${gradleBuildType} \
-      || exit 1
+      assemble${gradleBuildType}
+    '';
+  in
+    assert ANDROID_ABI_SPLIT != null && ANDROID_ABI_SPLIT != "";
+    assert stringLength ANDROID_ABI_INCLUDE > 0;
+  ''
+    # Fixes issue with failing to load libnative-platform.so
+    export GRADLE_USER_HOME=$(mktemp -d)
+    export ANDROID_SDK_HOME=$(mktemp -d)
+
+    echo "Adhoc ENV: ${adhocEnvVars}"
+    echo "Running: ${gradleCommand}"
+
+    pushd ./android
+    ${adhocEnvVars} ${gradleCommand} || exit 1
     popd > /dev/null
   '';
   doCheck = true;

nix/overlay.nix

@@ -13,11 +13,8 @@ in {
   # Fix for MacOS
   mkShell = super.mkShell.override { stdenv = stdenvNoCC; };
 
-  # Various utilities
-  utils = callPackage ./tools/utils.nix { };
   lib = (super.lib or { }) // (import ./lib {
     inherit (super) lib;
-    inherit (self) config;
   });
 
   # Project dependencies
@@ -29,9 +26,6 @@ in {
     react-native = callPackage ./deps/react-native { };
   };
 
-  # For parsing gradle.properties into an attrset
-  gradlePropParser = callPackage ./tools/gradlePropParser.nix { };
-
   # Fix for missing libarclite_macosx.a in Xcode 14.3.
   # https://github.com/ios-control/ios-deploy/issues/580
   ios-deploy = super.darwin.ios-deploy.overrideAttrs (old: rec {

nix/scripts/shell.sh

@@ -41,16 +41,6 @@ nixArgs=(
     "--attr shells.${TARGET}"
 )
 
-config=''
-if [[ -n "${STATUS_GO_SRC_OVERRIDE}" ]]; then
-    config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";"
-fi
-config+="status-im.build-type=\"${BUILD_TYPE}\";"
-
-if [[ -n "$config" ]]; then
-    nixArgs+=("--arg config {$config}")
-fi
-
 # This variable allows specifying which env vars to keep for Nix pure shell
 # The separator is a colon
 if [[ -n "${_NIX_KEEP}" ]]; then

nix/status-go/default.nix

@@ -1,7 +1,7 @@
 { lib, callPackage, mkShell }:
 
 let
-  inherit (lib) getConfig attrValues mapAttrs;
+  inherit (lib) getEnvWithDefault attrValues mapAttrs;
 
   # Metadata common to all builds of status-go
   meta = {
@@ -18,7 +18,7 @@ let
     GitCommit = source.rev;
     Version = source.cleanVersion;
     # FIXME: This should be moved to status-go config.
-    IpfsGatewayURL = getConfig "status-go.ipfs-gateway-url" "https://ipfs.status.im/";
+    IpfsGatewayURL = getEnvWithDefault "STATUS_GO_IPFS_GATEWAY_URL" "https://ipfs.status.im/";
   };
 
   # These are necessary for status-go to show correct version

nix/status-go/mobile/build.nix

@@ -1,4 +1,4 @@
-{ lib, utils, buildGoPackage
+{ callPackage, lib, buildGoPackage
 , androidPkgs, openjdk, gomobile, xcodeWrapper, removeReferencesTo
 , meta
 , source
@@ -13,6 +13,7 @@ let
   inherit (lib) concatStringsSep optionalString optional;
   isIOS = platform == "ios";
   isAndroid = platform == "android";
+  enforceXCodeAvailable = callPackage ./enforceXCodeAvailable.nix { };
 
 in buildGoPackage {
   pname = source.repo;
@@ -35,7 +36,7 @@ in buildGoPackage {
   ANDROID_HOME = optionalString isAndroid androidPkgs.sdk;
 
   # Ensure XCode is present for iOS, instead of failing at the end of the build.
-  preConfigure = optionalString isIOS utils.enforceXCodeAvailable;
+  preConfigure = optionalString isIOS enforceXCodeAvailable;
 
   buildPhase = ''
     runHook preBuild

nix/status-go/mobile/enforceXCodeAvailable.nix

@@ -0,0 +1,30 @@
+{ xcodeWrapper  }:
+
+let
+  RED = "\\033[0;31m";
+  GREEN = "\\033[0;32m";
+  NC = "\\033[0m";
+
+  _xcodeToolsTest = ''
+    xcode=0
+    iPhoneSDK=0
+    export PATH=${xcodeWrapper}/bin:$PATH
+    xcrun xcodebuild -version && xcode=1
+    [ $xcode -eq 1 ] && xcrun --sdk iphoneos --show-sdk-version > /dev/null && iPhoneSDK=1
+  '';
+  _xcodeToolReportScript = tool-name: ''[ $SELECTED -eq 0 ] && echo -e "${NC}- ${RED}[ ] ${tool-name}" || echo -e "${NC}- ${GREEN}[√] ${tool-name}${RED}"'';
+  _xcodeToolsReportScript = ''
+    echo -e "${RED}There are some required tools missing in the system:"
+    export SELECTED=$xcode; ${_xcodeToolReportScript "Xcode ${_xcodeVersion}"}
+    export SELECTED=$iPhoneSDK; ${_xcodeToolReportScript "iPhone SDK"}
+  '';
+  _xcodeVersion = builtins.replaceStrings ["xcode-wrapper-"] [""] xcodeWrapper.name;
+in
+  ''
+    ${_xcodeToolsTest}
+    if [ $xcode -eq 0 ]; then
+      ${_xcodeToolsReportScript}
+      echo -e "Please install Xcode ${_xcodeVersion} from the App Store.${NC}"
+      exit 1
+    fi
+  ''

nix/status-go/source.nix

@@ -1,9 +1,10 @@
-{ utils, lib, fetchFromGitHub }:
+{ lib, fetchFromGitHub }:
 
 let
-  inherit (lib) strings traceValFn importJSON getConfig;
+  inherit (lib) strings traceValFn importJSON getEnvWithDefault;
 
-  srcOverride = getConfig "status-go.src-override" null;
+  # TODO: move to default?
+  srcOverride = getEnvWithDefault "STATUS_GO_SRC_OVERRIDE" null;
   # Warning message about using local sources
   localSrcWarn = (path: "Using local status-go sources from ${path}");
 
@@ -43,7 +44,7 @@ let
     rev = versionJSON.commit-sha1;
     shortRev = strings.substring 0 7 rev;
     rawVersion = versionJSON.version;
-    cleanVersion = utils.sanitizeVersion versionJSON.version;
+    cleanVersion = lib.sanitizeVersion versionJSON.version;
     # Need to pretend this is from status-im to let Go build it.
     goPackagePath = "github.com/status-im/${repo}";
     src = fetchFromGitHub {