Merge branch 'release/v0.1.6'

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.6
+
+* Add [a Clojure frontend](https://github.com/Factual/drake/wiki/A-Clojure-Frontend-to-Drake) (thanks morrifeldman)
+* bugfix: [#129](https://github.com/Factual/drake/issues/129) (thanks calfzhou)
+
 ## 0.1.5
 
 * bugfix: [#98](https://github.com/Factual/drake/issues/98) --help now doesn't run workflow (thanks marshallshen)

README.md

@@ -26,7 +26,8 @@ We've not tested it on other operating systems.
 
 You can build Drake from source, which is the preferred way to run the most
 up-to-date version, or you can
-[download a prebuilt uberjar](https://docs.google.com/uc?export=download&confirm=0H_c&id=0B2xtKcFEL6wwc28ybDFQUWZBR3c)
+[download a prebuilt uberjar](https://docs.google.com/uc?export=download&confirm=nT8F&id=0B2xtKcFEL6wwWnRzVzRZcGFFaWc)
+
 ,which may not be the most recent version of Drake.
 
 Following are instructions for building from source. Drake is a Clojure project, so you will need to have [leiningen](https://github.com/technomancy/leiningen).

demos/clj-frontend/.gitignore

@@ -0,0 +1,9 @@
+/target
+/classes
+/checkouts
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/.lein-*
+/.nrepl-port

demos/clj-frontend/README.md

@@ -0,0 +1,29 @@
+# clj-frontend
+
+A demonstration leiningen project showing how to use drake.clj-frontend.
+
+## Usage
+
+Run `lein repl` from within the root directory of this project to start a
+repl and interact with the workflows found in the `clj.frontend.demo`
+namespace in `/src/clj_frontend/demo.clj`.
+
+For example, try these commands at the repl.
+
+```clojure
+(run-workflow minimal-workflow :preview true)
+
+(run-workflow minimal-workflow)
+
+(run-workflow advanced-workflow :preview true)
+
+(run-workflow advanced-workflow)
+
+(run-workflow reduce-workflow :preview true)
+```
+
+## License
+
+Copyright © 2014
+
+Distributed under the Eclipse Public License either version 1.0.

demos/clj-frontend/project.clj

@@ -0,0 +1,8 @@
+(defproject clj-frontend "0.1.0-SNAPSHOT"
+  :description "Demo project for drake.clj-frontend"
+  :url "http://example.com/FIXME"
+  :license {:name "Eclipse Public License"
+            :url "http://www.eclipse.org/legal/epl-v10.html"}
+  :dependencies [[org.clojure/clojure "1.5.1"]
+                 [factual/drake "0.1.5"]]
+  :repl-options {:init-ns clj-frontend.demo})

demos/clj-frontend/src/clj_frontend/demo.clj

@@ -0,0 +1,119 @@
+(ns clj-frontend.demo
+  (:use drake.clj-frontend))
+
+;; minimal workflow example
+
+;; Define a workflow called minimal-workflow.
+(def minimal-workflow
+  (->
+   (new-workflow)                       ;Create a new workflow
+   (cmd-step                            ;Add a command step with the
+                                        ;following arguments
+    ["out"]                             ;Array of outputs
+    []                                  ;Array of inputs
+    ["echo \"We are writing to a file here\" > $OUTPUT"] ;Array of commands
+    )))
+
+;; To see what will happen without running the workflow uncomment the
+;; following line or run it at in the repl:
+
+;; (run-workflow minimal-workflow :preview true)
+
+;; Use the following line to actually run the workflow
+
+;; (run-workflow minimal-workflow)
+
+
+;; A more advanced example
+
+(def advanced-workflow
+  (->
+   (new-workflow)
+   (cmd-step
+    ["out1"
+     "out2"]
+    []
+    ["echo \"This is the first output.\" > $OUTPUT0"
+     "echo \"This is the second output.\" > $OUTPUT1"] ;multiple commands
+    :timecheck false)                   ;options are key value pairs
+   (method
+    "test_method"
+    ["echo \"Here we are using a method.\" > $OUTPUT"])
+   (method-step
+    ["out_method"]                      ;outputs
+    []                                  ;inputs
+    "test_method")                      ;method name
+   (set-var "test_var" "TEST_VAR_VALUE") ;var name, var value
+   (set-var "output_three" "out3")
+   (cmd-step
+    ["$[output_three]"]                 ;inputs and outputs can have
+                                        ;$[XXX] substitution
+    ["out1" "%a_tag"]                   ;tags are allowed in inputs
+                                        ;and outputs
+    ;; $[XXX] substitution is allowed in commands.
+    ["echo \"This is the third output.\" > $OUTPUT"
+     "echo \"test_var is set to $test_var - $[test_var].\" >> $OUTPUT"
+     "echo \"The file $INPUT contains:\" | cat - $INPUT >> $[OUTPUT]"])))
+
+;; (run-workflow advanced-workflow :preview true)
+;; (run-workflow advanced-workflow)
+
+;; Example with reduce
+
+;; Let's say you want to take several raw data sources from the
+;; internet and for each source you want to create a directory,
+;; download some data into it, and do several processing steps on the
+;; data. We will express this as a map called dir->url-map between the
+;; directory names we want to create and the raw data sources we want
+;; to process.
+
+(def dir->url-map
+  "Hash map of:
+  Directory Names => URLs"
+  {"Dir1" "http://url1"
+   "Dir2" "http://url2"
+   "Dir3" "http://url3"})
+
+;; Now we need a function that takes an existing workflow and adds new
+;; steps to it for each directory => url pair from our data-map.
+
+
+(defn download-and-process
+  "I take an existing workflow and download and process the data at
+  url into the directory dir"
+  [w-flow [dir url]]                    ;note the argument
+                                        ;destructuring
+  (-> w-flow
+      (base "")                         ;make sure we are in top
+                                        ;directory
+      (cmd-step
+       [dir]
+       []
+       ["mkdir -p $OUTPUT"])
+      (base dir)                        ;move into dir for our
+                                        ;subsequent commands
+      (cmd-step
+       ["raw_data"]
+       []
+       ["wget -O $OUTPUT "  url]        ;get the data
+       :timecheck false)
+      (cmd-step
+       ["sorted_data"]
+       ["raw_data"]
+       ["sort -o $OUTPUT"])            ;sort the data
+      ;; more steps can be added here
+      ))
+
+;; Finally we can use `reduce` with `download-and-process` to add
+;; several workflow steps for each dir => url pair in dir->url-map.
+
+(def reduce-workflow
+  (reduce
+   download-and-process
+   (new-workflow)
+   dir->url-map))
+
+;; (run-workflow reduce-workflow :preview true)
+
+;; this is a fake workflow in that the interet data doesn't exist so
+;; we can't acutally run it

project.clj

@@ -11,6 +11,7 @@
                  [clojopts/clojopts "0.3.2"]
                  [fs "1.3.2"]
                  [factual/jlk-time "0.1"]
+                 [clj-time "0.6.0"]
                  [digest "1.4.0"]
                  [com.google.guava/guava "14.0.1"]
                  [cheshire "5.2.0"]

resources/regtest/regtest_command_substitution.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Regression testing for Drake: variable string substitution inside a command substitution
+# Relevant URLs:
+# https://github.com/Factual/drake/issues/129
+
+source $(dirname $0)/regtest_utils.sh
+
+cleanup() {
+  rm -f test_variable.out
+}
+
+echo "----------------------------"
+echo "TESTS: variable substitution"
+echo "----------------------------"
+
+cleanup
+run_d regtest_command_substitution_variable.d -a
+check "`cat test_variable.out`" '-world-/-world-/-$[VAR]-/-6-'
+
+# All tests passed
+echo "ALL PASSED"
+
+# Clean up again
+cleanup

resources/regtest/regtest_command_substitution_variable.d

@@ -0,0 +1,8 @@
+VAR="world"
+TEXT1=$(echo -$[VAR]-)
+TEXT2=$(echo -"$[VAR]"-)
+TEXT3=$(echo -'$[VAR]'-)
+TEXT4=$(echo -"$(echo $[VAR] | wc -c)"-)
+
+test_variable.out <-
+    echo $TEXT1/$TEXT2/$TEXT3/$TEXT4 > $OUTPUT

resources/regtest/run-all.sh

@@ -3,6 +3,7 @@ if ($(dirname $0)/regtest_fs.sh &&
     $(dirname $0)/regtest_stdout.sh &&
     $(dirname $0)/regtest_interpreters.sh &&
     $(dirname $0)/regtest_inline_shell.sh &&
+    $(dirname $0)/regtest_command_substitution.sh &&
     $(dirname $0)/regtest_async.sh &&
     $(dirname $0)/regtest_splitvars.sh &&
     $(dirname $0)/regtest_inputs_outputs.sh &&