Feature/optional files

src/drake/core.clj

@@ -101,13 +101,21 @@
     (assoc step :inputs branch-adjusted-inputs
                 :outputs branch-adjusted-outputs)))
 
-(defn- normalize-filename-for-run
+(defn- normalize-filename-for-run*
   "Normalizes filename and also removes local filesystem prefix (file:) from
    it. This is safe to do since it's the default filesystem,
    but it gives us a bit easier compatibility with existing tools."
   [filename]
   (let [n (dfs/normalized-path filename)]
-    (if (= "file" (dfs/path-fs n)) (dfs/path-filename n) n)))
+    (if (= "file" (dfs/path-fs n))
+      (dfs/path-filename n)
+      n)))
+
+(defn- normalize-filename-for-run
+  [filename]
+  (parser/modify-filename
+   filename
+   normalize-filename-for-run*))
 
 (defn- despace-cmds
   "Given a sequence of commands, removes leading whitespace found in the first
@@ -144,7 +152,7 @@
         normalized-outputs (map normalize-filename-for-run outputs)
         normalized-inputs (map normalize-filename-for-run inputs)
         vars (merge vars
-                    (parser/inouts-map normalized-inputs "INPUT")
+                    (parser/existing-inputs-map normalized-inputs "INPUT")
                     (parser/inouts-map normalized-outputs "OUTPUT"))
         method (methods (:method opts))
         method-mode (:method-mode opts)
@@ -176,6 +184,16 @@
       :vars vars
       :opts (if-not method opts (merge (:opts method) opts)))))
 
+(defn- existing-and-empty-inputs
+  "Remove '?' denoting optional file from front of path"
+  [inputs]
+  (let [inputs-info (map parser/make-file-stats inputs)]
+    {:existing (->> (filter :exists inputs-info)
+                    (map :file))
+     ;; Non-existing, non-optional inputs
+     :missing (->> (remove (some-fn :optional :exists) inputs-info)
+                   (map :file))}))
+
 (defn- should-build?
   "Given the parse tree and a step index, determines whether it should
    be built and returns the reason (e.g. 'timestamped') or
@@ -195,7 +213,7 @@
   [step forced triggered match-type fail-on-empty]
   (trace "should-build? fail-on-empty: " fail-on-empty)
   (let [{:keys [inputs outputs opts]} (branch-adjust-step step false)
-        empty-inputs (filter #(not (fs di/data-in? %)) inputs)
+        {inputs :inputs empty-inputs :missing} (existing-and-empty-inputs inputs)
         no-outputs (empty? outputs)]
     (trace "should-build? forced:" forced)
     (trace "should-build? match-type:" match-type)
@@ -320,16 +338,6 @@
    true if the step was actually run; false if skipped."
   [parse-tree step-number {:keys [index build match-type opts]}]
   (let [{:keys [inputs] :as step} (get-in parse-tree [:steps index])]
-    ;; TODO(artem)
-    ;; Somewhere here, before running the step or checking timestamps, we need to
-    ;; check for optional files and replace them with empty strings if they're
-    ;; not found (according to the spec). We shouldn't just rewrite :inputs and
-    ;; should probably separate two versions, since the step name
-    ;; (used in debugging and log files names) should not vary.
-    ;; For now just check that none of the input files is optional.
-    (if (some #(= \? (first %)) inputs)
-      (throw+ {:msg (str "optional input files are not supported yet: "
-                         inputs)}))
     (let [step-descr (step-string (branch-adjust-step step false))
           step (-> step
                    (update-in [:opts] merge opts)

src/drake/fs.clj

@@ -464,4 +464,4 @@
 
 (defn newest-in
   [path]
-  (pick-by-mod-time path -))
+  (pick-by-mod-time path -))

src/drake/parser.clj

@@ -3,10 +3,12 @@
             [clojure.string :as s]
             [clojure.core.memoize :as memo]
             [slingshot.slingshot :refer [throw+]]
+            [drake.fs :as dfs]
             [drake.shell :refer [shell]]
             [drake.steps :refer [add-dependencies calc-step-dirs]]
             [drake.utils :as utils :refer [clip ensure-final-newline]]
             [drake.parser_utils :refer :all]
+            [drake-interface.core :as di]
             [name.choi.joshua.fnparse :as p]
             [fs.core :as fs]))
 
@@ -297,14 +299,49 @@
                         second))]   ;; first is ",", second is <file-name>
    (cons first-file rest-files)))
 
-(defn add-prefix
+(def ^:private ^:const opt-flag \?)
+
+(defn optional-input?
+  "Check if the first character of a file specification is '?',
+   indicating it is an optional input"
+  [input]
+  (= opt-flag (first input)))
+
+(defn normalize-optional-file
+  [filename]
+  (if (optional-input? filename)
+    (subs filename 1)
+    filename))
+
+(defn make-file-stats
+  [filename]
+  (let [filepath (normalize-optional-file filename)]
+    {:optional (optional-input? filename)
+     :exists (dfs/fs di/data-in? filepath)
+     :path filepath
+     :file filename}))
+
+(defn modify-filename
+  [filename mod-fn]
+  (if (optional-input? filename)
+    (str opt-flag (mod-fn (normalize-optional-file filename)))
+    (mod-fn filename)))
+
+(defn- add-prefix*
   "Appends prefix if necessary (unless prepended by '!')."
   [prefix file]
   (cond (= \! (first file)) (clip file)
         (= \/ (first file)) file
         (re-matches #"[a-zA-z][a-zA-Z0-9+.-]*:.*" file) file
         :else (str prefix file)))
 
+(defn add-prefix
+  "add-prefix*, with support for file naming conventions"
+  [prefix file]
+  (modify-filename
+   file
+   (fn [f] (add-prefix* prefix f))))
+
 (defn add-path-sep-suffix [^String path]
   (if (or (empty? path)
           (.endsWith path "/")
@@ -339,6 +376,24 @@
         (for [[i v] (map-indexed vector items)]
           [(str prefix i) v])))
 
+(defn- fname->var
+  "Convert a file name to a var if it exists. If it
+   does not exist, and is not optional, use empty
+   string as var representation"
+  [filename]
+  (let [file-stats (make-file-stats filename)]
+    (if (and (not (:exists file-stats))
+             (:optional file-stats))
+      ""
+      (:path file-stats))))
+
+(defn existing-inputs-map
+  "Like inouts-map, except optional but nonexisting
+   input files are replaced by empty strings"
+  [items prefix]
+  (let [items (map fname->var items)]
+    (inouts-map items prefix)))
+
 ;; TODO(artem)
 ;; Should we move this to a common library?
 (defn demix

src/drake/parser_utils.clj

@@ -240,4 +240,4 @@
   (p/complex [_ single-quote
               chars (p/rep* (p/except p/anything single-quote))
               _ single-quote]
-             (apply str chars)))
+             (apply str chars)))