Compile nginx with libzauth using nix (#988)

* Compile nginx with libzauth using nix/niv

Noteworthy:

both vts and moreheaders are packaged in nixpkgs, so no need
to fetch those git submodules... but perhaps there are version
conflicts. we should check! :)

I also use niv for version management of nixpkgs. instead of hacky own
thing.

* Git ignore rust build artifacts

* Actually add rust build artifacts to gitignore.

It's in the top-level gitignore of wire-server, but not in the one in
libzauth

This is all a bit confusing. We should unify everything in one top-level
gitignore. That makes things a lot less complicated. However,
this is an impurity to tackle in another commit

libs/libzauth/.gitignore

@@ -1 +1,2 @@
 libzauth-c/deb/usr
+target

nix/default.nix

@@ -0,0 +1,14 @@
+let
+  sources = import ./sources.nix;
+  pkgs = import sources.nixpkgs {
+    config.allowUnfree = true;
+    overlays = [
+      # the tool we use for versioning (The thing that generates sources.json)
+      (_: _: { niv = (import sources.niv {}).niv; })
+      # All wire-server specific packages
+      (import ./overlays/wire-server.nix)
+
+    ];
+  };
+in
+  pkgs

nix/overlays/wire-server.nix

@@ -0,0 +1,60 @@
+self: super: {
+  # TODO: Do not use buildRustPackage. Ces't horrible
+  cryptobox = self.callPackage (
+    { fetchFromGitHub, rustPlatform, pkgconfig, libsodium }:
+      rustPlatform.buildRustPackage rec {
+        name = "cryptobox-c-${version}";
+        version = "2019-06-17";
+        buildInputs = [ pkgconfig libsodium ];
+        src = fetchFromGitHub {
+          owner = "wireapp";
+          repo = "cryptobox-c";
+          rev = "4067ad96b125942545dbdec8c1a89f1e1b65d013";
+          sha256 = "1i9dlhw0xk1viglyhail9fb36v1awrypps8jmhrkz8k1bhx98ci3";
+        };
+        cargoSha256 = "0m85c49hvvxxv7jdipfcaydy4n8iw4h6myzv63v7qc0fxnp1vfm8";
+        postInstall = ''
+          mkdir -p $out/include
+          cp src/cbox.h $out/include
+        '';
+      }
+  ) {};
+
+  zauth = self.callPackage (
+    { fetchFromGitHub, rustPlatform, pkgconfig, libsodium }:
+      rustPlatform.buildRustPackage rec {
+        name = "libzauth-${version}";
+        version = "3.0.0";
+        buildInputs = [ libsodium pkgconfig ];
+        src = self.nix-gitignore.gitignoreSource [] ../../libs/libzauth;
+
+        sourceRoot = "libzauth/libzauth-c";
+
+        cargoSha256 = "01yj1rchqmjnpj5cb9wl7vdzrycjwjhm60xh1jghw02n8jhl51p2"; # self.lib.fakeSha256;
+        postInstall = ''
+          mkdir -p $out/lib/pkgconfig
+          mkdir -p $out/include
+          cp src/zauth.h $out/include
+          sed -e "s~<<VERSION>>~${version}~" \
+            -e "s~<<PREFIX>>~$out~" \
+            src/libzauth.pc > $out/lib/pkgconfig/libzauth.pc
+          cp target/release/libzauth.so $out/lib/
+        '';
+      }
+  ) {};
+
+  nginxModules = super.nginxModules // {
+    zauth = {
+      src = ../../services/nginz/third_party/nginx-zauth-module;
+      inputs = [ self.pkg-config self.zauth ];
+    };
+  };
+
+  nginz = super.nginx.override {
+    modules = [
+      self.nginxModules.vts
+      self.nginxModules.moreheaders
+      self.nginxModules.zauth
+    ];
+  };
+}

nix/sources.json

@@ -0,0 +1,26 @@
+{
+    "niv": {
+        "branch": "master",
+        "description": "Easy dependency management for Nix projects",
+        "homepage": "https://github.com/nmattia/niv",
+        "owner": "nmattia",
+        "repo": "niv",
+        "rev": "50600603b51432839c4b6267fd6a0d00ae6b0451",
+        "sha256": "1rrhlscbqdn9a77ws49acl536n3mz6bai68z08mpg8qqa4ahr2sn",
+        "type": "tarball",
+        "url": "https://github.com/nmattia/niv/archive/50600603b51432839c4b6267fd6a0d00ae6b0451.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs": {
+        "branch": "nixos-19.09",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "8731aaaf8b30888bc24994096db830993090d7c4",
+        "sha256": "1hcc89rxi47nb0mpk05nl9rbbb04kfw97xfydhpmmgh57yrp3zqa",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/8731aaaf8b30888bc24994096db830993090d7c4.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,128 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = spec:
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; };
+
+  fetch_tarball = spec:
+    if spec.builtin or true then
+      builtins_fetchTarball { inherit (spec) url sha256; }
+    else
+      pkgs.fetchzip { inherit (spec) url sha256; };
+
+  fetch_git = spec:
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+
+  fetch_builtin-tarball = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-tarball" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=tarball -a builtin=true
+      ''
+      builtins_fetchTarball { inherit (spec) url sha256; };
+
+  fetch_builtin-url = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-url" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=file -a builtin=true
+      ''
+      (builtins_fetchurl { inherit (spec) url sha256; });
+
+  #
+  # The sources to fetch.
+  #
+
+  sources = builtins.fromJSON (builtins.readFile ./sources.json);
+
+  #
+  # Various helpers
+  #
+
+  # The set of packages used when specs are fetched using non-builtins.
+  pkgs =
+    if hasNixpkgsPath
+    then
+      if hasThisAsNixpkgsPath
+      then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
+      else import <nixpkgs> {}
+    else
+      import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
+
+  sources_nixpkgs =
+    if builtins.hasAttr "nixpkgs" sources
+    then sources.nixpkgs
+    else abort
+      ''
+        Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+        add a package called "nixpkgs" to your sources.json.
+      '';
+
+  hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
+  hasThisAsNixpkgsPath =
+    (builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
+
+  # The actual fetching function.
+  fetch = name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file spec
+    else if spec.type == "tarball" then fetch_tarball spec
+    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
+    else if spec.type == "builtin-url" then fetch_builtin-url spec
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball { inherit url; }
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl { inherit url; }
+      else
+        fetchurl attrs;
+
+in
+mapAttrs (
+  name: spec:
+    if builtins.hasAttr "outPath" spec
+    then abort
+      "The values in sources.json should not have an 'outPath' attribute"
+    else
+      spec // { outPath = fetch name spec; }
+) sources

services/integration.sh

@@ -106,9 +106,19 @@ run federator "" ${blue}
 
 function run_nginz() {
     colour=$1
-    prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
-    (cd ${NGINZ_WORK_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${TOP_LEVEL}/dist/nginx -p ${NGINZ_WORK_DIR} -c ${NGINZ_WORK_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
-        | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+
+    # For nix we dont need LD_LIBRARY_PATH; we link against libzauth directly.
+    # nix-build will put a symlink to ./result with the nginx artifact
+    if which nix-build; then
+      nginz=$(nix-build "${TOP_LEVEL}/nix" -A nginz --no-out-link )
+      (cd ${NGINZ_WORK_DIR} && ${nginz}/bin/nginx -p ${NGINZ_WORK_DIR} -c ${NGINZ_WORK_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+          | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+    else
+      prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
+
+      (cd ${NGINZ_WORK_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${TOP_LEVEL}/dist/nginx -p ${NGINZ_WORK_DIR} -c ${NGINZ_WORK_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+          | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+    fi
 }
 
 NGINZ_PORT=""

shell.nix

@@ -1,7 +1,7 @@
-{ pkgs ? import <nixpkgs> {}}:
+{ pkgs ? import ./nix }:
 with pkgs; mkShell {
   name = "shell";
-  buildInputs = [ 
+  buildInputs = [
     docker-compose
     gnumake
     stack

stack-deps.nix

@@ -1,56 +1,20 @@
 let
-  # Pin nixpkgs for all dependencies.
-  # If you want to update.
-  # 1. go to https://nixos.org/channels/nixos-19.09
-  # 2. copy the URL to nixexprs.tar.gz and the sha256 hash
-  # 3. Uncomment the  sha256 = 00000 field
-  # 4. nix-build
-  # 5. Make nix complain to you what the correct hash is.
-  # 6. comment sha256 = 0000 and   add sha256 = <actual-hash>
-  # 7. nix-build
-  # 8. commit
-  # TODO(arianvp): There are tools that automate this; we should use them
-  pkgsTar = builtins.fetchTarball {
-    name = "nixos-1909";
-    url = "https://releases.nixos.org/nixos/19.09/nixos-19.09.1019.c5aabb0d603/nixexprs.tar.xz";
-    sha256 = "1hjw843g964aj9cd9p6x5473yy4sfmqnqlvavc5c1lbqa8v676zg";
-    # sha256 = "0000000000000000000000000000000000000000000000000000";
-  };
-  pkgs = import pkgsTar {};
-  cryptobox-c = pkgs.callPackage ({fetchFromGitHub, rustPlatform,  pkgconfig, libsodium}:
-    rustPlatform.buildRustPackage rec {
-      name = "cryptobox-c-${version}";
-      version = "2019-06-17";
-      buildInputs = [ pkgconfig libsodium ];
-      src = fetchFromGitHub {
-        owner = "wireapp";
-        repo = "cryptobox-c";
-        rev = "4067ad96b125942545dbdec8c1a89f1e1b65d013";
-        sha256 = "1i9dlhw0xk1viglyhail9fb36v1awrypps8jmhrkz8k1bhx98ci3";
-      };
-      cargoSha256 = "0m85c49hvvxxv7jdipfcaydy4n8iw4h6myzv63v7qc0fxnp1vfm8";
-      postInstall = ''
-        mkdir -p $out/include
-        cp src/cbox.h $out/include
-      '';
-    }) {};
-  hoogle = pkgs.haskellPackages.hoogle;
+  pkgs = import ./nix;
 in
-  pkgs.haskell.lib.buildStackProject {
-    name = "wire-server";
-    buildInputs = with pkgs; [
-      cryptobox-c
-      geoip
-      git
-      icu
-      libsodium
-      libxml2
-      openssl
-      pkgconfig
-      protobuf
-      snappy
-      zlib
-      hoogle
-    ];
-    ghc = pkgs.haskell.compiler.ghc865;
-  }
+pkgs.haskell.lib.buildStackProject {
+  name = "wire-server";
+  buildInputs = with pkgs; [
+    cryptobox
+    geoip
+    git
+    icu
+    libsodium
+    libxml2
+    openssl
+    pkgconfig
+    protobuf
+    snappy
+    zlib
+  ];
+  ghc = pkgs.haskell.compiler.ghc865;
+}