Skip to content

Commit

Permalink
Refactor setup for generated nix files
Browse files Browse the repository at this point in the history
the reiterates on #1220. The motivation for the refactoring was that the
generated files were not using `gitSource`, so I would get cache misses
and dirty files in the repo. Fixing this was easier if all generated nix
files are in one place, so I did the following changes:

 * All generated nix files are in `nix/generated`. This de-pollutes the
   normal working area with generated files, and keeps them out from
   sources (more precise derivations).
 * The local generated files now use `gitSource`.
 * The generator is now `nix/generate.nix`.
 * I moved complex logic from `nix/default.nix` to `nix/generate.nix`,
   this means everything is in one place, instead of spread over several
   nix files.
 * There is now only a single check to check if the generated files are
   upto date, and a single command to update the files.
 * This check is no longer marked `allowSubstitutes = false`. The check
    isn’t cheap (it requires `cabal2nix`) and we want the check to happen
    on builders, not hydra itself, so that the cache has `cabal2nix`.
 * The check is now included in `all-systems-go`.
  • Loading branch information
nomeata committed Feb 20, 2020
1 parent b90fbc5 commit c4b943d
Show file tree
Hide file tree
Showing 12 changed files with 125 additions and 107 deletions.
22 changes: 7 additions & 15 deletions Building.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,20 +81,12 @@ https://github.com/ocaml/dune/issues/57 to see when a coverage report is viable
## Updating Haskell Packages

When the `.cabal` file of a Haskell package is changed you need to make sure the
corresponding `default.nix` file (stored in the same directory as the `.cabal`
file) is kept in sync with it.
corresponding `default.nix` file (stored in `nix/generated/`) is kept in sync
with it.

As mentioned in the `default.nix` files, these files are automatically generated:
As mentioned in the `nix/generate.nix` files, these files are automatically
generated. See `nix/generate.nix` for the command to update them.

```
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
```

Note that they also contain the instructions on how to update them.

Don't worry if you forget to update the `default.nix` file. There are CI jobs
(like [`haskellSrc2nix.lsp-int`](https://hydra.dfinity.systems/job/dfinity-ci-build/motoko/haskellSrc2nix.lsp-int)
that check if the files are in sync and fail with a diff if they aren't.
Don't worry if you forget to update the `default.nix` file, the CI job
`check-generated` checks if these files are in sync and fail with a diff if
they aren't.
27 changes: 9 additions & 18 deletions default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -411,24 +411,14 @@ rec {
};
stdlib-adocs = stdlib-doc.adocs;

haskellSrc2nix =
let
check = dir: nixpkgs.runCommandNoCC "check-${builtins.baseNameOf dir}" {
preferLocalBuild = true;
allowSubstitutes = false;
nativeBuildInputs = [ nixpkgs.diffutils ];
expected = import (dir + "/generate.nix") { pkgs = nixpkgs; };
inherit dir;
} ''
diff -U 3 $expected/default.nix $dir/default.nix
touch $out
'';
in {
lsp-int = check ./test/lsp-int;
qc-motoko = check ./test/random;
ic-stub = check ./ic-stub;
winter = check ./winter;
};
check-generated = nixpkgs.runCommandNoCC "check-generated" {
nativeBuildInputs = [ nixpkgs.diffutils ];
expected = import ./nix/generate.nix { pkgs = nixpkgs; };
dir = ./nix/generated;
} ''
diff -r -U 3 $expected $dir
touch $out
'';

all-systems-go = nixpkgs.releaseTools.aggregate {
name = "all-systems-go";
Expand All @@ -447,6 +437,7 @@ rec {
users-guide
ic-stub
shell
check-generated
] ++ builtins.attrValues tests
++ builtins.attrValues examples;
};
Expand Down
1 change: 0 additions & 1 deletion ic-stub/generate.nix

This file was deleted.

40 changes: 0 additions & 40 deletions nix/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -25,46 +25,6 @@ let
};
}
)
(self: super: {
# A variant of `haskellSrc2nixWithDoc` for local Haskell packages.
localHaskellSrc2nixWithDoc = name: path: extraCabal2nixOptions:
self.haskellSrc2nixWithDoc {
inherit name extraCabal2nixOptions;
src = import ./gitSource.nix path;
src_subst = "./.";
};

# `haskellSrc2nixWithDoc` is used to generate `default.nix` files for
# Haskell packages which are intended to be stored in the repository.
#
# The function generates a directory containing a `default.nix` which
# is the result of running `cabal2nix` with the `extraCabal2nixOptions`
# on the provided `src`.
#
# A header is added to `default.nix` which contains instructions on
# how to regenerate that file.
#
# Finally the `src` attribute in the `default.nix` will be defined as
# `src_subst` such that it can be pointed to local or niv-managed
# sources.
haskellSrc2nixWithDoc = {name, src, src_subst, extraCabal2nixOptions}:
let
drv = pkgs.haskellPackages.haskellSrc2nix {
inherit name extraCabal2nixOptions src;
};
in drv.overrideAttrs (oldAttrs: {
message = ''
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!\
# To regenerate this file execute the following command in this directory:\
#\
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
'';
inherit src_subst;
installPhase = oldAttrs.installPhase + ''
sed -i "1i$message;s|src = .*|src = $src_subst;|" $out/default.nix
'';
});
})
];
};
in
Expand Down
75 changes: 75 additions & 0 deletions nix/generate.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# This file generates the contents of nix/generated/. Use
#
# cp -fv $(nix-build generate.nix --no-link)/ generated/
#
# to update

{ pkgs ? import ../nix {} }:

let

# `haskellSrc2nixWithDoc` is used to generate `default.nix` files for
# Haskell packages which are intended to be stored in the repository.
#
# The function generates a directory containing a `default.nix` which
# is the result of running `cabal2nix` with the `extraCabal2nixOptions`
# on the provided `src`.
#
# A header is added to `default.nix` which contains instructions on
# how to regenerate that file.
#
# Finally the `src` attribute in the `default.nix` will be defined as
# `src_subst` such that it can be pointed to local or niv-managed
# sources.
haskellSrc2nixWithDoc = {name, src, src_subst, extraCabal2nixOptions}:
let
drv = pkgs.haskellPackages.haskellSrc2nix {
inherit name extraCabal2nixOptions src;
};
in drv.overrideAttrs (oldAttrs: {
message = ''
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!\
# See ./nix/generate.nix for instructions.\
'';
inherit src_subst;
installPhase = oldAttrs.installPhase + ''
sed -i "1i$message;s|src = .*|src = $src_subst;|" $out/default.nix
# Accept `pkgs` as an argument in case the `src_subst` depends on it.
sed -i "s|{ mkDerivation|{ mkDerivation, pkgs|" $out/default.nix
'';
});

# A variant of `haskellSrc2nixWithDoc` for local Haskell packages.
localHaskellSrc2nixWithDoc = name: path: extraCabal2nixOptions:
haskellSrc2nixWithDoc {
inherit name extraCabal2nixOptions;
src = import ./gitSource.nix path;
src_subst = "import ../gitSource.nix \"${path}\"";
};


winter = (haskellSrc2nixWithDoc {
name = "winter";
src = pkgs.sources.winter;
src_subst = "pkgs.sources.winter";
extraCabal2nixOptions = "--no-check";
});

ic-stub = localHaskellSrc2nixWithDoc "ic-stub" "ic-stub" "--no-check";
random = localHaskellSrc2nixWithDoc "qc-motoko" "test/random" "";
lsp-int = localHaskellSrc2nixWithDoc "lsp-int" "test/lsp-int" "";

allGenerated = pkgs.runCommandNoCC "generated" {} ''
mkdir -p $out
cp ${winter}/default.nix $out/winter.nix
cp ${ic-stub}/default.nix $out/ic-stub.nix
cp ${random}/default.nix $out/random.nix
cp ${lsp-int}/default.nix $out/lsp-int.nix
'';
in
allGenerated




9 changes: 4 additions & 5 deletions ic-stub/default.nix → nix/generated/ic-stub.nix
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
{ mkDerivation, base, binary, bytestring, containers
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, binary, bytestring, containers
, data-default-class, filepath, hex-text, mtl, optparse-applicative
, primitive, stdenv, text, transformers, utf8-string, vector
, winter
}:
mkDerivation {
pname = "ic-stub";
version = "0.1.0.0";
src = ./.;
src = import ../gitSource.nix "ic-stub";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
Expand Down
20 changes: 20 additions & 0 deletions nix/generated/lsp-int.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, data-default, directory, filepath
, haskell-lsp-types, hspec, HUnit, lens, lsp-test, stdenv, text
}:
mkDerivation {
pname = "lsp-int";
version = "0";
src = import ../gitSource.nix "test/lsp-int";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
base data-default directory filepath haskell-lsp-types hspec HUnit
lens lsp-test text
];
description = "Integration tests for the language server";
license = "unknown";
hydraPlatforms = stdenv.lib.platforms.none;
}
9 changes: 4 additions & 5 deletions test/random/default.nix → nix/generated/random.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
{ mkDerivation, base, exceptions, managed, process, QuickCheck
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, base, exceptions, managed, process, QuickCheck
, quickcheck-text, quickcheck-unicode, stdenv, tasty
, tasty-quickcheck, text, turtle
}:
mkDerivation {
pname = "qc-motoko";
version = "1";
src = ./.;
src = import ../gitSource.nix "test/random";
isLibrary = false;
isExecutable = true;
executableHaskellDepends = [
Expand Down
5 changes: 2 additions & 3 deletions winter/default.nix → nix/generated/winter.nix
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# THIS IS AN AUTOMATICALLY GENERATED FILE. DO NOT EDIT MANUALLY!
# To regenerate this file execute the following command in this directory:
#
# cp $(nix-build ./generate.nix --no-link)/default.nix ./default.nix
# See ./nix/generate.nix for instructions.

{ mkDerivation, pkgs, array, base, binary, byte-order, bytestring
, cmdargs, containers, data-default-class, data-fix, deepseq
, directory, filepath, FloatingHex, lifted-base, microlens-platform
Expand Down
8 changes: 4 additions & 4 deletions nix/haskell-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ nix: subpath:

lsp-test = nix.haskell.lib.dontCheck self.lsp-test_0_10_1_0;

lsp-int = super.callPackage ../test/lsp-int {};
lsp-int = super.callPackage generated/lsp-int.nix {};

qc-motoko = super.callPackage ../test/random {};
qc-motoko = super.callPackage generated/random.nix {};

winter = super.callPackage ../winter {};
winter = super.callPackage generated/winter.nix {};

ic-stub = super.callPackage ../ic-stub {};
ic-stub = super.callPackage generated/ic-stub.nix {};
}
1 change: 0 additions & 1 deletion test/random/generate.nix

This file was deleted.

15 changes: 0 additions & 15 deletions winter/generate.nix

This file was deleted.

0 comments on commit c4b943d

Please sign in to comment.