Rewrite from the ground-up.

I spent the last couple of months rewriting Kevlar from the ground up.
There was a lot of experimentation and throw-away code in the Git
history so I decided to squash everything in one giant commit.

The most noticeable change is that now Kevlar uses the Dhall
configuration language for specifying the build steps.  This choice
guided me towards a  much simpler implementation.

[1] https://dhall-lang.org/
[2] http://wiki.c2.com/?PlanToThrowOneAway

.gitignore

@@ -4,3 +4,6 @@ kevlar.cabal
 
 _build
 _shake
+
+# Dhall
+.history

.kevlar.dhall

@@ -0,0 +1,57 @@
+let ci = ./dhall/ci.dhall
+
+let bakeBuilderImage =
+      ci.bakeDockerImage
+      "kevlar-builder"
+      ./docker/kevlar-builder/Dockerfile as Text
+
+let bakePublishImage =
+      ci.bakeDockerImage
+      "kevlar-publish"
+      ./docker/kevlar-publish/Dockerfile as Text
+
+let build =
+        λ(src : Text)
+      →   ci.Step
+        ⫽ { name =
+              "build"
+          , script =
+              ./ci/build.sh as Text
+          , image =
+              ci.loadFromStep bakeBuilderImage
+          , need =
+              [ ci.fetch { src = src, name = "src" }
+              , ci.output "${bakeBuilderImage.name}"
+              ]
+          , caches =
+              [ ".stack" ]
+          }
+
+let publish =
+        ci.Step
+      ⫽ { name =
+            "publish"
+        , script =
+            ./ci/publish.sh as Text
+        , image =
+            ci.loadFromStep bakePublishImage
+        , need =
+            [ ci.output "${bakePublishImage.name}", ci.output "build" ]
+        , caches =
+            [ ".stack" ]
+        , params =
+            [ { name =
+                  "GITHUB_ACCESS_TOKEN"
+              , value =
+                  env:GITHUB_ACCESS_TOKEN as Text ? ""
+              }
+            , { name =
+                  "KEVLAR_VERSION"
+              , value =
+                  env:KEVLAR_VERSION as Text ? ""
+              }
+            ]
+        }
+
+in    λ(repo : Text)
+    → { steps = [ bakeBuilderImage, build repo, bakePublishImage, publish ] }

ChangeLog.md

@@ -2,6 +2,10 @@
 
 ## Unreleased changes
 
+- A complete rewrite of the initial implementation
+- YAML configuration format is replaced by Dhall
+- The support for handling secrets using `pass` is temporarilty dropped
+
 ## [0.0.1] - 2019-01-10
 
 - Initial implementation.

README.md

@@ -5,15 +5,24 @@ Kevlar is an experimental CI/CD system.
 ## Example
 
 The `example` subdirectory contains a small project.  Its build rules can be
-found in the [configuration file](example/.kevlar/config.yml).  You can try
+found in the [configuration file](example/.kevlar.dhall).  You can try
 running:
 
     cd example
     kevlar test
 
 Kevlar is self-hosting, that is it can build itself.  Take a look at [its own
-configuration file](.kevlar/config.yml)
+configuration file](.kevlar.dhall)
 
 ## Design
 
 There's a [design document](design.md) explaining the basic concepts of Kevlar.
+
+
+## Release
+
+The release script requires some enviroment variables to be set. The command:
+
+    GITHUB_ACCESS_TOKEN=$(pass api/github.com/kevlar-deployment) VERSION=v0.0.2 kevlar publish
+
+creates a draft release on Github.

app/Main.hs

@@ -1,30 +1,32 @@
-module Main where
+{-# LANGUAGE OverloadedStrings #-}
 
-import           Kevlar
-import           Kevlar.Pipeline
-import           Kevlar.Yaml
+module Main where
 
-import           Control.Monad                  ( forM_ )
+import Control.Monad (forM_)
+import Data.Text as T
+import Development.Shake
+import System.IO.Temp
 
-import           Development.Shake
+import Kevlar
+import Kevlar.Config
+import Kevlar.Git
+import Kevlar.Step
 
 kevlarConfig :: FilePath
-kevlarConfig = ".kevlar/config.yml"
+kevlarConfig = ".kevlar.dhall"
 
 main :: IO ()
-main = shakeArgs shakeOptions { shakeFiles = "_build" } $ do
-  rulesOracle
-
-  phony "clean" $ do
-    putNormal "Cleaning artifacts in _build"
-    removeFilesAfter "_build/artifacts" ["//*"]
-
-  phony "purge" $ do
-    putNormal "Removing _build"
-    removeFilesAfter "_build" ["//*"]
-
-  pipeline <- liftIO $ readPipeline kevlarConfig
-  forM_ (steps pipeline) $ \step -> do
-    mkRules step
-
-    phony (name step) $ need [kevlarConfig, build (name step)]
+main = withSystemTempDirectory "kevlar" shakeMain
+
+shakeMain :: FilePath -> IO ()
+shakeMain src =
+  shakeArgs shakeOptions {shakeFiles = "_build"} $ do
+    phony "clean" $ do
+      putNormal "Cleaning artifacts in _build"
+      removeFilesAfter "_build/artifacts" ["//*"]
+    phony "purge" $ do
+      putNormal "Removing _build"
+      removeFilesAfter "_build" ["//*"]
+    config <- liftIO $ readConfig kevlarConfig
+    forM_ (steps . config $ T.pack src) mkRules
+    liftIO $ copyGitFiles src

ci/build.sh

@@ -3,4 +3,4 @@ set -e
 (cd src && stack install \
     --test \
     --local-bin-path "$KEVLAR_OUTPUT" \
-    --ghc-options "-optl-static -fPIC -optc-Os")
+    --flag=kevlar:static)

dhall/bakeDockerImage.dhall

@@ -0,0 +1,18 @@
+let types = ./../dhall/types.dhall
+
+let defaults = ./../dhall/defaults.dhall
+
+in    λ(image : Text)
+    → λ(dockerfile : Text)
+    →     defaults.Step
+        ⫽ { name =
+              "${image}"
+          , image =
+              None types.Image
+          , script =
+              ''
+              docker build -t ${image} - <<< "${dockerfile}"
+              docker save -o output/${image}.tar ${image}
+              ''
+          }
+      : types.Step

dhall/ci.dhall

@@ -0,0 +1,18 @@
+{- A high-level module to be used in CI pipeline definitions -}
+let types = ./../dhall/types.dhall
+
+let defaults = ./../dhall/defaults.dhall
+
+let output = λ(name : Text) → types.Need.Output { name = name }
+
+in  { Step =
+        defaults.Step
+    , bakeDockerImage =
+        ./../dhall/bakeDockerImage.dhall
+    , loadFromStep =
+        ./../dhall/loadFromStep.dhall
+    , fetch =
+        types.Need.Fetch
+    , output =
+        output
+    }

dhall/defaults.dhall

@@ -0,0 +1,26 @@
+let types = ./types.dhall
+
+let alpine = { name = "alpine", load = None Text } : types.Image
+
+in  { Config =
+        { steps = [] : List types.Step }
+    , Step =
+          { name =
+              "hello"
+          , shell =
+              "/bin/bash"
+          , script =
+              "echo 'hello, kevlar'"
+          , image =
+              Some alpine
+          , need =
+              [] : List types.Need
+          , caches =
+              [] : List Text
+          , params =
+              [] : List types.Param
+          }
+        : types.Step
+    , Image =
+        alpine
+    }

dhall/fetch.dhall

@@ -0,0 +1,5 @@
+let types = ./types.dhall
+
+in λ(src : Text) → λ(dst : Text) → { _1 = src, _2 = dst } : types.Input
+
+

dhall/loadFromStep.dhall

@@ -0,0 +1,14 @@
+let defaults = ./defaults.dhall
+
+let types = ./types.dhall
+
+in    λ(step : types.Step)
+    →   Some
+        (   defaults.Image
+          ⫽ { name =
+                "${step.name}"
+            , load =
+                Some "${step.name}/${step.name}.tar"
+            }
+        )
+      : Optional types.Image

dhall/types.dhall

@@ -0,0 +1,27 @@
+{- Kevlar standard library -}
+let Image = { name : Text, load : Optional Text }
+
+let Need = < Fetch : { src : Text, name : Text } | Output : { name : Text } >
+
+let Param = { name : Text, value : Text }
+
+let Step =
+      { name :
+          Text
+      , shell :
+          Text
+      , script :
+          Text
+      , image :
+          Optional Image
+      , need :
+          List Need
+      , caches :
+          List Text
+      , params :
+          List Param
+      }
+
+let Config = { steps : List Step }
+
+in  { Config = Config, Image = Image, Need = Need, Step = Step, Param = Param }

docker/kevlar-builder/Dockerfile

@@ -1,18 +1,14 @@
-FROM ubuntu:18.10
+FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y\
     build-essential \
     curl \
     libffi-dev \
     libgmp-dev \
+    libtinfo-dev \
     xz-utils \
     zlib1g-dev \
  && rm -rf /var/lib/apt/lists/*
 
 RUN curl -L https://get.haskellstack.org/stable/linux-x86_64.tar.gz \
   | tar xz --wildcards --strip-components=1 -C /usr/local/bin '*/stack'
-
-# workaround described in
-# https://www.fpcomplete.com/blog/2016/10/static-compilation-with-stack
-RUN (cd /usr/lib/gcc/x86_64-linux-gnu/8  \
-    && cp crtbeginS.o crtbeginT.o --backup --suffix=.orig)

example/.kevlar.dhall

@@ -0,0 +1,61 @@
+let ci = ./../dhall/ci.dhall
+
+let bakeBuilderImage =
+      ci.bakeDockerImage
+      "hello-world-builder"
+      ''
+      FROM alpine
+      RUN apk update && apk add build-base
+      ''
+
+let bakeTesterImage =
+      ci.bakeDockerImage
+      "hello-world-tester"
+      ''
+      FROM alpine
+      RUN apk update && apk add tree
+      ''
+
+let build =
+        λ(src : Text)
+      →   ci.Step
+        ⫽ { name =
+              "build"
+          , script =
+              ./build.sh as Text
+          , image =
+              ci.loadFromStep bakeBuilderImage
+          , need =
+              [ ci.fetch { src = src, name = "src" }
+              , ci.output "${bakeBuilderImage.name}"
+              ]
+          }
+
+let test =
+        ci.Step
+      ⫽ { name =
+            "test"
+        , script =
+            ''
+            #!/bin/sh
+            set -e
+            echo "Artifacts from previous steps are available in this container"
+            tree -L 1
+
+            echo
+            echo "Environment variables can be specified in the step: \$HELLO=$HELLO"
+
+            echo
+            echo "Running the output artifacts from the 'build' step"
+            build/hello
+            ''
+        , image =
+            ci.loadFromStep bakeTesterImage
+        , need =
+            [ ci.output "build", ci.output "${bakeTesterImage.name}" ]
+        , params =
+            [ { name = "HELLO", value = "world!" } ]
+        }
+
+in    λ(repo : Text)
+    → { steps = [ build repo, test, bakeBuilderImage, bakeTesterImage ] }