Merge branch 'release/v0.1.5'

CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.1.5
+
+* bugfix: [#98](https://github.com/Factual/drake/issues/98) --help now doesn't run workflow (thanks marshallshen)
+* Upgrade to c4 0.2.0, which no longer bundles the Facebook API
+* Basic functionality working for Windows sytems, specifically, Windows 8/command shell.
+* bugfix: [#11](https://github.com/Factual/drake/issues/111) FileSystem plugins get wired up properly (thanks derenrich)
+* Initial fix for [#118](https://github.com/Factual/drake/issues/118) to handle quotes better in shell commands  (thanks myronahn)
+* Fixes and documentation for core.run-workflow
+* Add node (Javascript) protocol (thanks arowla)
+* Upgrade c4 to version 0.2.0, which drops bundling of Facebook Places API support
+* Supprot for command line var regex
+
 ## 0.1.4
 
  * Added support for async execution of steps via --jobs (thanks guillaume and myronahn). See [Async Execution of Steps](https://github.com/Factual/drake/wiki/Async-Execution-of-Steps)

README.md

@@ -19,13 +19,14 @@ If you like screencasts, check out this [Drake walk-through video](http://www.yo
 
 ## Installation
 
-You will need to have a JVM installed. Drake has been tested under Linux and Mac OS X. We've not tested it on Windows.
+You will need to have a JVM installed. Drake has been tested under Linux, Mac OS X and Windows 8.
+We've not tested it on other operating systems.
 
 ### Download or build the uberjar
 
 You can build Drake from source, which is the preferred way to run the most
 up-to-date version, or you can
-[download a prebuilt uberjar](https://docs.google.com/uc?export=download&confirm=gS92&id=0B2xtKcFEL6wwdlNEWi1QdjlnSEU)
+[download a prebuilt uberjar](https://docs.google.com/uc?export=download&confirm=0H_c&id=0B2xtKcFEL6wwc28ybDFQUWZBR3c)
 ,which may not be the most recent version of Drake.
 
 Following are instructions for building from source. Drake is a Clojure project, so you will need to have [leiningen](https://github.com/technomancy/leiningen).
@@ -144,7 +145,7 @@ There are annotated workflow examples in the demos directory.
 
 There's a [Google Group for Drake](https://groups.google.com/forum/?fromgroups#!forum/drake-workflow) where you can ask questions. And if you found a bug or want to submit a feature request, go to [Drake's GitHub issues page](https://github.com/Factual/drake/issues?sort=created&state=open).
 
-## Asynchronous Excecution of Steps
+## Asynchronous Execution of Steps
 
 Please see [the wiki page on async](https://github.com/Factual/drake/wiki/Async-Execution-of-Steps).
 
@@ -175,6 +176,8 @@ Thanks to [Lars Yencken](https://github.com/larsyencken), we now have [Vim synta
 
 <img src="https://lh3.googleusercontent.com/-mqNpFqf7P0k/UQoXkpAqr1I/AAAAAAAAADU/U5zrvozVmzE/s400/image.png"/>
 
+Also thanks to [Lars Yencken](https://github.com/larsyencken), [utilities for making life easier in Python with Drake workflows](https://pypi.python.org/pypi/drakeutil).
+
 Courtesy of [@daguar](https://gist.github.com/daguar), an [alternative approach to installing Drake on Mac OS X](https://gist.github.com/daguar/5368778).
 
 ## License

project.clj

@@ -1,21 +1,24 @@
-(defproject factual/drake "0.1.4"
+(defproject factual/drake "0.1.5"
   :description "Drake: the data processing workflow tool (a.k.a. 'make for data')"
   :url "https://github.com/Factual/drake"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.4.0"]
+                 [org.clojure/core.memoize "0.5.6"]
                  [factual/drake-interface "0.0.1"]
                  [org.clojure/tools.logging "0.2.3"]
                  [clj-logging-config "1.9.6"]
                  [clojopts/clojopts "0.3.2"]
                  [fs "1.3.2"]
                  [factual/jlk-time "0.1"]
                  [digest "1.4.0"]
-                 [slingshot "0.10.2"]
+                 [com.google.guava/guava "14.0.1"]
+                 [cheshire "5.2.0"]
+                 [slingshot "0.10.3"]
                  [factual/fnparse "2.3.0"]
                  [commons-codec/commons-codec "1.6"]
                  [factual/sosueme "0.0.15"]
-                 [factual/c4 "0.1.2"]
+                 [factual/c4 "0.2.0"]
                  ;; for HDFS support
                  [hdfs-clj "0.1.0"]
                  ;; you may need to change this to be compatible with your cluster

release-checklist.md

@@ -6,4 +6,5 @@
   * Update README uberjar link
   * Update README feature docs
   * Update wiki feature docs
+  * Create release label
   * Notify newsgroup

resources/regtest/regtest_inline_shell.sh

@@ -12,6 +12,8 @@ cleanup() {
   for FILE in $FILES
   do
     rm -f $(dirname $0)/$FILE>/dev/null 2>&1
+    rm -f date1
+    rm -f date2
   done
 }
 
@@ -30,8 +32,20 @@ for FILE in $FILES
 do
 check_grep $(dirname $0)/$FILE "$FILE"
 done
+
+# Make sure memoized shell runs do not prevent running a shell command multiple times
+# https://github.com/Factual/drake/issues/118
+
+echo "-----------"
+echo "TESTS: memo"
+echo "-----------"
+
+run_d regtest_inline_shell_memo.d -a
+check_not_equal `cat date1` `cat date2`
+
 echo "ALL PASSED"
 
+
 # Clean up again
 cleanup
 

resources/regtest/regtest_inline_shell_memo.d

@@ -0,0 +1,8 @@
+DATE1:=$(sleep 1; python -c "import datetime; print datetime.datetime.now()")
+DATE2:=$(sleep 1; python -c "import datetime; print datetime.datetime.now()")
+
+date1 <-
+  echo $DATE1 > date1
+
+date2 <-
+  echo $DATE2 > date2

resources/regtest/regtest_interpreters.sh

@@ -15,4 +15,4 @@ check_grep $(dirname $0)/python.out "python_processed:test_line"
 check_grep $(dirname $0)/ruby.out "ruby_processed:test_line"
 check_grep $(dirname $0)/R.out "R_processed:test_line"
 
-echo "ALL PASSED"
+echo "ALL PASSED"

resources/regtest/regtest_s3.sh

@@ -91,6 +91,7 @@ check_local merged "L1L2S1S2"
 echo "------------------"
 echo "TEST: merged_local"
 echo "------------------"
+sleep 2 # get around the 1 second accuracy of lastModified() in Java
 create_local_file merged_local "LL"
 run
 check_targets 1
@@ -101,6 +102,7 @@ check_local merged "LLS1S2"
 echo "-----------------"
 echo "TEST: merged_s3"
 echo "-----------------"
+sleep 2 # get around the 1 second accuracy of lastModified() in Java
 create_s3_file ${S3_TEST_DIR}/merged_s3 "SS"
 run
 check_targets 1
@@ -111,6 +113,7 @@ check_local merged "LLSS"
 echo "-------------------"
 echo "TEST: change s3_2"
 echo "-------------------"
+sleep 2 # get around the 1 second accuracy of lastModified() in Java
 create_s3_file ${S3_TEST_DIR}/s3_2 "s2"
 run
 check_targets 2
@@ -121,6 +124,7 @@ check_local merged "LLS1s2"
 echo "--------------------"
 echo "TEST: change local_2"
 echo "--------------------"
+sleep 2 # get around the 1 second accuracy of lastModified() in Java
 create_local_file local_2 "l2"
 run
 check_targets 2

resources/regtest/regtest_splitvars.d

@@ -0,0 +1,3 @@
+$[FILE2] <-
+  echo -n "$OUTPUT" > $OUTPUT
+

resources/regtest/regtest_splitvars.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Regression testing for Drake: splitting --vars by regex
+# Relevant URLs:
+# https://github.com/Factual/drake/issues/97
+
+source $(dirname $0)/regtest_utils.sh
+
+export FILE=f1f1f1f1
+
+cleanup() {
+  rm -f $(dirname $0)/$FILE>/dev/null 2>&1
+}
+
+echo "-----------------"
+echo "TESTS: split vars"
+echo "-----------------"
+
+# First cleanup any existing files
+cleanup
+
+# Run test with default regex
+run_d regtest_splitvars.d -a --vars "RANDOMVAR=1,FILE2=$FILE"
+check_grep $(dirname $0)/$FILE "$FILE"
+cleanup
+
+# Run test with crazy split regex
+run_d regtest_splitvars.d -a --split-vars-regex "[a-c]ebopalul[a-z]" --vars "RANDOMVAR=1bebopalulaFILE2=$FILE"
+check_grep $(dirname $0)/$FILE "$FILE"
+cleanup
+
+echo "ALL PASSED"
+

resources/regtest/regtest_target_check_bug.d

@@ -0,0 +1,13 @@
+dude.txt <-
+  echo "$OUTPUT" > $OUTPUT
+
+babe.txt <- dude.txt
+  echo "$OUTPUT" > $OUTPUT
+  echo "belle.txt" > belle.txt
+
+belle.txt <- dude.txt
+  echo "$OUTPUT" > $OUTPUT
+
+bogus.txt <- babe.txt, belle.txt
+  cat $INPUTS > $OUTPUT
+  echo "$OUTPUT" > $OUTPUT

resources/regtest/regtest_target_check_bug.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Regression testing for Drake: target check but
+# This is a bug that manifests itself when a step is run
+# and the target is already built, but this was not detected
+# at the beginning during predict-steps
+#
+# Relevant URLs:
+
+source $(dirname $0)/regtest_utils.sh
+
+export FILES=dude.txt\ babe.txt\ belle.txt\ bogus.txt
+
+cleanup() {
+  for FILE in $FILES
+  do
+    rm -f $(dirname $0)/$FILE>/dev/null 2>&1
+  done
+}
+
+echo "-------------------"
+echo "TESTS: target check"
+echo "-------------------"
+
+# First cleanup any existing files
+cleanup
+
+# Run loop test
+run_d regtest_target_check_bug.d -a
+
+# Check results
+for FILE in $FILES
+do
+check_grep $(dirname $0)/$FILE "$FILE"
+done
+echo "ALL PASSED"
+
+# Clean up again
+cleanup
+

resources/regtest/regtest_utils.sh

@@ -10,6 +10,15 @@ check() {
   echo "PASS"
 }
 
+check_not_equal() {
+  if [ "$1" == "$2" ]; then
+    echo "FAIL"
+    echo "\"$1\" == \"$2\""
+    exit -1
+  fi
+  echo "PASS"
+}
+
 check_grep() {
   if ! grep "$2" "$1" >/dev/null; then
     echo "FAIL"

resources/regtest/run-all.sh

@@ -4,6 +4,7 @@ if ($(dirname $0)/regtest_fs.sh &&
     $(dirname $0)/regtest_interpreters.sh &&
     $(dirname $0)/regtest_inline_shell.sh &&
     $(dirname $0)/regtest_async.sh &&
+    $(dirname $0)/regtest_splitvars.sh &&
     $(dirname $0)/regtest_inputs_outputs.sh &&
     $(dirname $0)/regtest_methods.sh &&
     $(dirname $0)/regtest_protocol_eval.sh &&