From 94a87913eac90b38dec110b753f75476d6d33193 Mon Sep 17 00:00:00 2001 From: intractable Date: Sun, 21 Jan 2018 17:46:36 -0600 Subject: [PATCH] Update to nixpkgs 17.09 (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Deterministically pin to a 17.09 nixpkgs * Remove out-of-date cabal2nix output * `proto3-wire` → `d492fa3` * Fix shebangs for test scripts * Remove test scripts patchfile We no longer need to patch in this manner, as we will ensure that the correct GHC is on the nix-shell $PATH. * Don't barf on unset PYTHONPATH * Relax `aeson` bound * Simplify and cleanup toplevel nix * rm `nixpkgs.json` * Workaround High Sierra linking issue by restructuring ... the `nix-shell` GHC vs. that used by `testHaskellDepends`. --- default.nix | 1 + nix/aeson-pretty.nix | 20 --- nix/aeson.nix | 32 ---- nix/cabal-doctest.nix | 10 -- nix/insert-ordered-containers.nix | 23 --- nix/neat-interpolation.nix | 15 -- nix/optparse-applicative.nix | 15 -- nix/optparse-generic.nix | 14 -- nix/proto3-wire.nix | 6 +- nix/turtle.nix | 20 --- nixpkgs/17_09.nix | 12 ++ .../fetch-nixpkgs.nix | 2 +- proto3-suite.cabal | 4 +- release.nix | 151 ++++++++---------- tests/TestCodeGen.hs | 26 ++- tests/decode.sh | 4 +- tests/encode.sh | 4 +- tests/tests.patch | 30 ---- 18 files changed, 105 insertions(+), 284 deletions(-) delete mode 100644 nix/aeson-pretty.nix delete mode 100644 nix/aeson.nix delete mode 100644 nix/cabal-doctest.nix delete mode 100644 nix/insert-ordered-containers.nix delete mode 100644 nix/neat-interpolation.nix delete mode 100644 nix/optparse-applicative.nix delete mode 100644 nix/optparse-generic.nix delete mode 100644 nix/turtle.nix create mode 100644 nixpkgs/17_09.nix rename fetch-nixpkgs.nix => nixpkgs/fetch-nixpkgs.nix (99%) delete mode 100644 tests/tests.patch diff --git a/default.nix b/default.nix index e4f292f9..f0480e19 100644 --- a/default.nix +++ b/default.nix @@ -12,6 +12,7 @@ mkDerivation { src = ./.; isLibrary = true; isExecutable = true; + enableSeparateDataOutput = true; libraryHaskellDepends = [ aeson aeson-pretty attoparsec base base64-bytestring bytestring cereal containers deepseq foldl haskell-src lens mtl diff --git a/nix/aeson-pretty.nix b/nix/aeson-pretty.nix deleted file mode 100644 index fe695fa7..00000000 --- a/nix/aeson-pretty.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ mkDerivation, aeson, attoparsec, base, base-compat, bytestring -, cmdargs, scientific, stdenv, text, unordered-containers, vector -}: -mkDerivation { - pname = "aeson-pretty"; - version = "0.8.2"; - sha256 = "1c5r1w1hcv297pmj9yjpz9al22k3mh61gimi37wddga02212kd3c"; - isLibrary = true; - isExecutable = true; - libraryHaskellDepends = [ - aeson base base-compat bytestring scientific text - unordered-containers vector - ]; - executableHaskellDepends = [ - aeson attoparsec base bytestring cmdargs - ]; - homepage = "http://github.com/informatikr/aeson-pretty"; - description = "JSON pretty-printing library and command-line tool"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/aeson.nix b/nix/aeson.nix deleted file mode 100644 index da66e754..00000000 --- a/nix/aeson.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ mkDerivation, attoparsec, base, base-compat, base-orphans -, base16-bytestring, bytestring, containers, deepseq, directory -, dlist, filepath, generic-deriving, ghc-prim, hashable -, hashable-time, HUnit, integer-logarithms, QuickCheck -, quickcheck-instances, scientific, stdenv, tagged -, template-haskell, test-framework, test-framework-hunit -, test-framework-quickcheck2, text, time, time-locale-compat -, unordered-containers, uuid-types, vector -}: -mkDerivation { - pname = "aeson"; - version = "1.1.1.0"; - sha256 = "1mkj4a09x9psmgq9sg5nz9va76756zfm97ds2gk2qpgxc7nr2dq8"; - revision = "2"; - editedCabalFile = "10bc20f8807990e71f5db74a1b7029f81f888c6f9d1c03e93883555fd1291e84"; - libraryHaskellDepends = [ - attoparsec base base-compat bytestring containers deepseq dlist - ghc-prim hashable scientific tagged template-haskell text time - time-locale-compat unordered-containers uuid-types vector - ]; - testHaskellDepends = [ - attoparsec base base-compat base-orphans base16-bytestring - bytestring containers directory dlist filepath generic-deriving - ghc-prim hashable hashable-time HUnit integer-logarithms QuickCheck - quickcheck-instances scientific tagged template-haskell - test-framework test-framework-hunit test-framework-quickcheck2 text - time time-locale-compat unordered-containers uuid-types vector - ]; - homepage = "https://github.com/bos/aeson"; - description = "Fast JSON parsing and encoding"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/cabal-doctest.nix b/nix/cabal-doctest.nix deleted file mode 100644 index 360c9a1c..00000000 --- a/nix/cabal-doctest.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ mkDerivation, base, Cabal, directory, filepath, stdenv }: -mkDerivation { - pname = "cabal-doctest"; - version = "1.0.2"; - sha256 = "0h3wsjf2mg8kw1zvxc0f9nzchj5kzvza9z0arcyixkd9rkgqq6sa"; - libraryHaskellDepends = [ base Cabal directory filepath ]; - homepage = "https://github.com/phadej/cabal-doctest"; - description = "A Setup.hs helper for doctests running"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/insert-ordered-containers.nix b/nix/insert-ordered-containers.nix deleted file mode 100644 index 5f957133..00000000 --- a/nix/insert-ordered-containers.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ mkDerivation, aeson, base, base-compat, hashable, lens -, QuickCheck, semigroupoids, semigroups, stdenv, tasty -, tasty-quickcheck, text, transformers, unordered-containers -}: -mkDerivation { - pname = "insert-ordered-containers"; - version = "0.2.1.0"; - sha256 = "1612f455dw37da9g7bsd1s5kyi84mnr1ifnjw69892amyimi47fp"; - revision = "3"; - editedCabalFile = "6fdce987672b006226243aa17522b57ec7a9e1cab247802eddbdaa9dc5b06446"; - libraryHaskellDepends = [ - aeson base base-compat hashable lens semigroupoids semigroups text - transformers unordered-containers - ]; - testHaskellDepends = [ - aeson base base-compat hashable lens QuickCheck semigroupoids - semigroups tasty tasty-quickcheck text transformers - unordered-containers - ]; - homepage = "https://github.com/phadej/insert-ordered-containers#readme"; - description = "Associative containers retating insertion order for traversals"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/neat-interpolation.nix b/nix/neat-interpolation.nix deleted file mode 100644 index 06f6ba54..00000000 --- a/nix/neat-interpolation.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ mkDerivation, base, base-prelude, HTF, parsec, stdenv -, template-haskell, text -}: -mkDerivation { - pname = "neat-interpolation"; - version = "0.3.2.1"; - sha256 = "0550dy0vwh81byi9bxhdzqx5y9lnvkwj5rbks5rbj2fylhyf8c2m"; - libraryHaskellDepends = [ - base base-prelude parsec template-haskell text - ]; - testHaskellDepends = [ base-prelude HTF ]; - homepage = "https://github.com/nikita-volkov/neat-interpolation"; - description = "A quasiquoter for neat and simple multiline text interpolation"; - license = stdenv.lib.licenses.mit; -} diff --git a/nix/optparse-applicative.nix b/nix/optparse-applicative.nix deleted file mode 100644 index bdfc252a..00000000 --- a/nix/optparse-applicative.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ mkDerivation, ansi-wl-pprint, base, bytestring, process -, QuickCheck, stdenv, transformers, transformers-compat -}: -mkDerivation { - pname = "optparse-applicative"; - version = "0.14.0.0"; - sha256 = "06iwp1qsq0gjhnhxwyhdhldwvhlgcik6lx5jxpbb40fispyk4nxm"; - libraryHaskellDepends = [ - ansi-wl-pprint base process transformers transformers-compat - ]; - testHaskellDepends = [ base bytestring QuickCheck ]; - homepage = "https://github.com/pcapriotti/optparse-applicative"; - description = "Utilities and combinators for parsing command line options"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/optparse-generic.nix b/nix/optparse-generic.nix deleted file mode 100644 index 42bc1302..00000000 --- a/nix/optparse-generic.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ mkDerivation, base, bytestring, optparse-applicative, semigroups -, stdenv, system-filepath, text, time, transformers, void -}: -mkDerivation { - pname = "optparse-generic"; - version = "1.2.1"; - sha256 = "1dk945dp98mwk1v4y0cky3z0ngmd29nbg6fbaaxnigcrgpbvkjml"; - libraryHaskellDepends = [ - base bytestring optparse-applicative semigroups system-filepath - text time transformers void - ]; - description = "Auto-generate a command-line parser for your datatype"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nix/proto3-wire.nix b/nix/proto3-wire.nix index 305b82e0..1fd3032d 100644 --- a/nix/proto3-wire.nix +++ b/nix/proto3-wire.nix @@ -6,9 +6,9 @@ mkDerivation { pname = "proto3-wire"; version = "1.0.0"; src = fetchgit { - url = "https://github.com/awakenetworks/proto3-wire"; - sha256 = "14n0d16an782ayipirm5v2mvp58jgf65xvffqzp08p50sksil3gi"; - rev = "a938330bf794cf3fa05591d03906915df98d157c"; + url = "https://github.com/awakenetworks/proto3-wire.git"; + sha256 = "0nlar9zwy7k47nm395h11ivmhhfhip5bhyazwa2gnffn5lhsyv3i"; + rev = "d492fa3034724b46f23fb2c73780c9dd7ecb4d04"; }; libraryHaskellDepends = [ base bytestring cereal containers deepseq hashable QuickCheck safe diff --git a/nix/turtle.nix b/nix/turtle.nix deleted file mode 100644 index 4f667138..00000000 --- a/nix/turtle.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ mkDerivation, ansi-wl-pprint, async, base, bytestring, clock -, directory, doctest, foldl, hostname, managed, optional-args -, optparse-applicative, process, semigroups, stdenv, stm -, system-fileio, system-filepath, temporary, text, time -, transformers, unix, unix-compat -}: -mkDerivation { - pname = "turtle"; - version = "1.3.6"; - sha256 = "0fr8p6rnk2lrsgbfh60jlqcjr0nxrh3ywxsj5d4psck0kgyhvg1m"; - libraryHaskellDepends = [ - ansi-wl-pprint async base bytestring clock directory foldl hostname - managed optional-args optparse-applicative process semigroups stm - system-fileio system-filepath temporary text time transformers unix - unix-compat - ]; - testHaskellDepends = [ base doctest system-filepath temporary ]; - description = "Shell programming, Haskell-style"; - license = stdenv.lib.licenses.bsd3; -} diff --git a/nixpkgs/17_09.nix b/nixpkgs/17_09.nix new file mode 100644 index 00000000..5295b386 --- /dev/null +++ b/nixpkgs/17_09.nix @@ -0,0 +1,12 @@ +# Given a Git revision hash ``, you get the new SHA256 by running: +# +# ```bash +# $ nix-prefetch-url "https://github.com/NixOS/nixpkgs/archive/.tar.gz" +# ``` +# +# The SHA256 will be printed as the last line of stdout. + +import ./fetch-nixpkgs.nix { + rev = "74286ec9e76be7cd00c4247b9acb430c4bd9f1ce"; + sha256 = "0njb3qd2wxj7gil8y61lwh7zacmvr6zklv67w5zmvifi1fvalvdg"; +} diff --git a/fetch-nixpkgs.nix b/nixpkgs/fetch-nixpkgs.nix similarity index 99% rename from fetch-nixpkgs.nix rename to nixpkgs/fetch-nixpkgs.nix index 8b30bbc6..644b892c 100644 --- a/fetch-nixpkgs.nix +++ b/nixpkgs/fetch-nixpkgs.nix @@ -27,7 +27,7 @@ ifThenElse { }; builtin-paths = import ; - + script = builtins.toFile "nixpkgs-unpacker" '' "$coreutils/mkdir" "$out" cd "$out" diff --git a/proto3-suite.cabal b/proto3-suite.cabal index 09e34e0e..cafae14e 100644 --- a/proto3-suite.cabal +++ b/proto3-suite.cabal @@ -24,7 +24,7 @@ library other-modules: Proto3.Suite.DotProto.Internal Proto3.Suite.DotProto.Generate.Swagger Proto3.Suite.JSONPB.Class - build-depends: aeson == 1.1.1.0, + build-depends: aeson >= 1.1.1.0 && < 1.2, aeson-pretty, attoparsec >= 0.13.0.1, base >=4.8 && <5.0, @@ -66,7 +66,7 @@ test-suite tests default-language: Haskell2010 build-depends: base >=4.8 && <5.0, QuickCheck >=2.8 && <2.10, - aeson == 1.1.1.0, + aeson >= 1.1.1.0 && < 1.2, attoparsec >= 0.13.0.1, base >=4.8 && <5.0, base64-bytestring >= 1.0.0.1 && < 1.1, diff --git a/release.nix b/release.nix index 4a099071..be9e031a 100644 --- a/release.nix +++ b/release.nix @@ -1,99 +1,90 @@ -# To develop with this repository, open a Nix shell with: +# To develop iteratively within this repository, open a Nix shell via: # # $ nix-shell -A proto3-suite.env release.nix # -# ... then run `cabal` commands as you would normally do: +# ... and then use `cabal` to build and test: # -# [nix-shell]$ cabal configure --with-gcc=clang --enable-tests +# [nix-shell]$ cabal configure --enable-tests # [nix-shell]$ cabal build # [nix-shell]$ cabal test let - config = { - packageOverrides = pkgs: - let - python_protobuf3_0 = - (pkgs.pythonPackages.protobufBuild pkgs.protobuf3_0).override { - doCheck = false; - }; - in - { haskellPackages = pkgs.haskellPackages.override { - overrides = haskellPackagesNew: haskellPackagesOld: rec { - - aeson = - pkgs.haskell.lib.dontCheck (haskellPackagesNew.callPackage ./nix/aeson.nix { }); - - aeson-pretty = - haskellPackagesNew.callPackage ./nix/aeson-pretty.nix { }; - - cabal-doctest = - haskellPackagesNew.callPackage ./nix/cabal-doctest.nix { }; - - insert-ordered-containers = - haskellPackagesNew.callPackage ./nix/insert-ordered-containers.nix { }; - - neat-interpolation = - haskellPackagesNew.callPackage ./nix/neat-interpolation.nix { }; - - optparse-applicative = - haskellPackagesNew.callPackage ./nix/optparse-applicative.nix { } ; - - optparse-generic = - haskellPackagesNew.callPackage ./nix/optparse-generic.nix { } ; - - proto3-suite-no-tests = - pkgs.haskell.lib.dontCheck - (haskellPackagesNew.callPackage ./default.nix { }); - - proto3-suite = - pkgs.haskell.lib.overrideCabal - (haskellPackagesNew.callPackage ./default.nix { }) - (oldAttrs: { - patches = [ tests/tests.patch ]; - - postPatch = '' - substituteInPlace tests/encode.sh --replace @ghc@ ${pkgs.ghc} --replace @bash@ ${pkgs.bash} - substituteInPlace tests/decode.sh --replace @ghc@ ${pkgs.ghc} --replace @bash@ ${pkgs.bash} - ''; - - testHaskellDepends = oldAttrs.testHaskellDepends ++ [ - pkgs.ghc - proto3-suite-no-tests - pkgs.protobuf3_0 - pkgs.python - python_protobuf3_0 - ]; - } - ); - - proto3-wire = - haskellPackagesNew.callPackage ./nix/proto3-wire.nix { }; - - scientific = - pkgs.haskell.lib.dontCheck haskellPackagesOld.scientific; - - swagger2 = - pkgs.haskell.lib.dontHaddock (haskellPackagesNew.callPackage ./nix/swagger2.nix { }); - - turtle = - haskellPackagesNew.callPackage ./nix/turtle.nix { } ; - - }; - }; - }; + nixpkgs = import ./nixpkgs/17_09.nix; + config = { allowUnfree = true; - }; - nixpkgs = import ./fetch-nixpkgs.nix { - rev = "7ae9da426924537755ce9164fd5b5f81ce16a1c3"; - sha256 = "1zg1j2fsfj6qhh724nc8fnn3ig3rm6nd29zzhjrbjkszkjspc9pl"; + packageOverrides = pkgs: { + haskellPackages = pkgs.haskellPackages.override { + overrides = self: super: rec { + + # The test suite for proto3-suite requires: + # + # - a GHC with `proto3-suite` installed, since our code generation + # tests compile and run generated code; since this custom GHC is + # also used inside the nix-shell environment for iterative + # development, we ensure that it is available on $PATH and that + # all test suite deps are available to it + # + # - a Python interpreter with a protobuf package installed, which we + # use as a reference implementation; we also put expose this on + # the `nix-shell` $PATH + # + # Finally, we make `cabal` available in the `nix-shell`, intentionally + # occluding any globally-installed versions of the tool. + + proto3-suite = + pkgs.haskell.lib.overrideCabal + (self.callPackage ./default.nix { }) + (drv: + let + python = pkgs.python.withPackages (pkgs: [ pkgs.protobuf3_0 ]); + in + { + shellHook = (drv.shellHook or "") + + (let + ghc = self.ghcWithPackages (pkgs: + drv.testHaskellDepends ++ [ pkgs.proto3-suite-boot ] + ); + in '' + export PATH=${self.cabal-install}/bin:${ghc}/bin:${python}/bin''${PATH:+:}$PATH + ''); + + testHaskellDepends = drv.testHaskellDepends ++ [ + pkgs.ghc + proto3-suite-boot + python + ]; + } + ); + + # A proto3-suite sans tests, for bootstrapping + proto3-suite-boot = + pkgs.haskell.lib.overrideCabal + (self.callPackage ./default.nix { }) + (_: { + configureFlags = [ "--disable-optimization" ]; + doCheck = false; + doHaddock = false; + }); + + proto3-wire = + self.callPackage ./nix/proto3-wire.nix { }; + + swagger2 = + pkgs.haskell.lib.dontCheck + (pkgs.haskell.lib.dontHaddock + (self.callPackage ./nix/swagger2.nix { })); + }; + }; + }; }; +in +let linuxPkgs = import nixpkgs { inherit config; system = "x86_64-linux" ; }; darwinPkgs = import nixpkgs { inherit config; system = "x86_64-darwin"; }; pkgs = import nixpkgs { inherit config; }; - in { proto3-suite-linux = linuxPkgs.haskellPackages.proto3-suite; proto3-suite-darwin = darwinPkgs.haskellPackages.proto3-suite; diff --git a/tests/TestCodeGen.hs b/tests/TestCodeGen.hs index 3d0a3e0d..3a8f853e 100644 --- a/tests/TestCodeGen.hs +++ b/tests/TestCodeGen.hs @@ -67,21 +67,21 @@ don'tAlterEnumFieldNames $ prefixedEnumFieldName enumName fieldName @?= Right (enumName <> fieldName) +setPythonPath :: IO () +setPythonPath = Turtle.export "PYTHONPATH" =<< do + maybe pyTmpDir ((pyTmpDir <> ":") <>) <$> Turtle.need "PYTHONPATH" + simpleEncodeDotProto :: TestTree simpleEncodeDotProto = testCase "generate code for a simple .proto and then use it to encode messages" $ do compileTestDotProtos + -- Compile our generated encoder (@?= ExitSuccess) =<< Turtle.proc "tests/encode.sh" [hsTmpDir] empty - m <- Turtle.need "PYTHONPATH" - pythonPath <- case m of - Nothing -> fail "PYTHONPATH environment variable is not set" - Just pythonPath -> return pythonPath - Turtle.export "PYTHONPATH" (pythonPath <> ":" <> pyTmpDir) - - let cmd = (hsTmpDir <> "/simpleEncodeDotProto | python tests/check_simple_dot_proto.py") - -- The python test exits with a special error code to indicate all tests - -- were successful + -- The python encoder test exits with a special error code to indicate + -- all tests were successful + setPythonPath + let cmd = hsTmpDir <> "/simpleEncodeDotProto | python tests/check_simple_dot_proto.py" (@?= ExitFailure 12) =<< Turtle.shell cmd empty -- Not using bracket so that we can inspect the output to fix the tests @@ -92,14 +92,10 @@ simpleDecodeDotProto :: TestTree simpleDecodeDotProto = testCase "generate code for a simple .proto and then use it to decode messages" $ do compileTestDotProtos + -- Compile our generated decoder (@?= ExitSuccess) =<< Turtle.proc "tests/decode.sh" [hsTmpDir] empty - m <- Turtle.need "PYTHONPATH" - pythonPath <- case m of - Nothing -> fail "PYTHONPATH environment variable is not set" - Just pythonPath -> return pythonPath - Turtle.export "PYTHONPATH" (pythonPath <> ":" <> pyTmpDir) - + setPythonPath let cmd = "python tests/send_simple_dot_proto.py | " <> hsTmpDir <> "/simpleDecodeDotProto " (@?= ExitSuccess) =<< Turtle.shell cmd empty diff --git a/tests/decode.sh b/tests/decode.sh index 9ae39191..516134b3 100755 --- a/tests/decode.sh +++ b/tests/decode.sh @@ -1,5 +1,5 @@ -#!/bin/bash -eu - +#!/usr/bin/env bash +set -eu hsTmpDir=$1 ghc \ diff --git a/tests/encode.sh b/tests/encode.sh index 91fbd17a..9e94a4a0 100755 --- a/tests/encode.sh +++ b/tests/encode.sh @@ -1,5 +1,5 @@ -#!/bin/bash -eu - +#!/usr/bin/env bash +set -eu hsTmpDir=$1 ghc \ diff --git a/tests/tests.patch b/tests/tests.patch deleted file mode 100644 index ab76588f..00000000 --- a/tests/tests.patch +++ /dev/null @@ -1,30 +0,0 @@ -diff --git a/tests/decode.sh b/tests/decode.sh -index 59ad7cc..d1df62b 100755 ---- a/tests/decode.sh -+++ b/tests/decode.sh -@@ -1,8 +1,8 @@ --#!/bin/bash -eu -+#! @bash@/bin/bash -eu - - hsTmpDir=$1 - --ghc \ -+@ghc@/bin/ghc \ - --make \ - -odir $hsTmpDir \ - -hidir $hsTmpDir \ -diff --git a/tests/encode.sh b/tests/encode.sh -index 8ebea7e..bc79de2 100755 ---- a/tests/encode.sh -+++ b/tests/encode.sh -@@ -1,8 +1,8 @@ --#!/bin/bash -eu -+#! @bash@/bin/bash -eu - - hsTmpDir=$1 - --ghc \ -+@ghc@/bin/ghc \ - --make \ - -odir $hsTmpDir \ - -hidir $hsTmpDir \