diff --git a/src/drake/core.clj b/src/drake/core.clj index 83a4066..603e539 100644 --- a/src/drake/core.clj +++ b/src/drake/core.clj @@ -101,13 +101,21 @@ (assoc step :inputs branch-adjusted-inputs :outputs branch-adjusted-outputs))) -(defn- normalize-filename-for-run +(defn- normalize-filename-for-run* "Normalizes filename and also removes local filesystem prefix (file:) from it. This is safe to do since it's the default filesystem, but it gives us a bit easier compatibility with existing tools." [filename] (let [n (dfs/normalized-path filename)] - (if (= "file" (dfs/path-fs n)) (dfs/path-filename n) n))) + (if (= "file" (dfs/path-fs n)) + (dfs/path-filename n) + n))) + +(defn- normalize-filename-for-run + [filename] + (parser/modify-filename + filename + normalize-filename-for-run*)) (defn- despace-cmds "Given a sequence of commands, removes leading whitespace found in the first @@ -144,7 +152,7 @@ normalized-outputs (map normalize-filename-for-run outputs) normalized-inputs (map normalize-filename-for-run inputs) vars (merge vars - (parser/inouts-map normalized-inputs "INPUT") + (parser/existing-inputs-map normalized-inputs "INPUT") (parser/inouts-map normalized-outputs "OUTPUT")) method (methods (:method opts)) method-mode (:method-mode opts) @@ -176,6 +184,16 @@ :vars vars :opts (if-not method opts (merge (:opts method) opts))))) +(defn- existing-and-empty-inputs + "Remove '?' denoting optional file from front of path" + [inputs] + (let [inputs-info (map parser/make-file-stats inputs)] + {:existing (->> (filter :exists inputs-info) + (map :file)) + ;; Non-existing, non-optional inputs + :missing (->> (remove (some-fn :optional :exists) inputs-info) + (map :file))})) + (defn- should-build? "Given the parse tree and a step index, determines whether it should be built and returns the reason (e.g. 'timestamped') or @@ -195,7 +213,7 @@ [step forced triggered match-type fail-on-empty] (trace "should-build? fail-on-empty: " fail-on-empty) (let [{:keys [inputs outputs opts]} (branch-adjust-step step false) - empty-inputs (filter #(not (fs di/data-in? %)) inputs) + {inputs :inputs empty-inputs :missing} (existing-and-empty-inputs inputs) no-outputs (empty? outputs)] (trace "should-build? forced:" forced) (trace "should-build? match-type:" match-type) @@ -320,16 +338,6 @@ true if the step was actually run; false if skipped." [parse-tree step-number {:keys [index build match-type opts]}] (let [{:keys [inputs] :as step} (get-in parse-tree [:steps index])] - ;; TODO(artem) - ;; Somewhere here, before running the step or checking timestamps, we need to - ;; check for optional files and replace them with empty strings if they're - ;; not found (according to the spec). We shouldn't just rewrite :inputs and - ;; should probably separate two versions, since the step name - ;; (used in debugging and log files names) should not vary. - ;; For now just check that none of the input files is optional. - (if (some #(= \? (first %)) inputs) - (throw+ {:msg (str "optional input files are not supported yet: " - inputs)})) (let [step-descr (step-string (branch-adjust-step step false)) step (-> step (update-in [:opts] merge opts) diff --git a/src/drake/fs.clj b/src/drake/fs.clj index 2377ba6..893eb48 100644 --- a/src/drake/fs.clj +++ b/src/drake/fs.clj @@ -464,4 +464,4 @@ (defn newest-in [path] - (pick-by-mod-time path -)) \ No newline at end of file + (pick-by-mod-time path -)) diff --git a/src/drake/parser.clj b/src/drake/parser.clj index 3ac4ab4..93706b6 100644 --- a/src/drake/parser.clj +++ b/src/drake/parser.clj @@ -3,10 +3,12 @@ [clojure.string :as s] [clojure.core.memoize :as memo] [slingshot.slingshot :refer [throw+]] + [drake.fs :as dfs] [drake.shell :refer [shell]] [drake.steps :refer [add-dependencies calc-step-dirs]] [drake.utils :as utils :refer [clip ensure-final-newline]] [drake.parser_utils :refer :all] + [drake-interface.core :as di] [name.choi.joshua.fnparse :as p] [fs.core :as fs])) @@ -297,7 +299,35 @@ second))] ;; first is ",", second is (cons first-file rest-files))) -(defn add-prefix +(def ^:private ^:const opt-flag \?) + +(defn optional-input? + "Check if the first character of a file specification is '?', + indicating it is an optional input" + [input] + (= opt-flag (first input))) + +(defn normalize-optional-file + [filename] + (if (optional-input? filename) + (subs filename 1) + filename)) + +(defn make-file-stats + [filename] + (let [filepath (normalize-optional-file filename)] + {:optional (optional-input? filename) + :exists (dfs/fs di/data-in? filepath) + :path filepath + :file filename})) + +(defn modify-filename + [filename mod-fn] + (if (optional-input? filename) + (str opt-flag (mod-fn (normalize-optional-file filename))) + (mod-fn filename))) + +(defn- add-prefix* "Appends prefix if necessary (unless prepended by '!')." [prefix file] (cond (= \! (first file)) (clip file) @@ -305,6 +335,13 @@ (re-matches #"[a-zA-z][a-zA-Z0-9+.-]*:.*" file) file :else (str prefix file))) +(defn add-prefix + "add-prefix*, with support for file naming conventions" + [prefix file] + (modify-filename + file + (fn [f] (add-prefix* prefix f)))) + (defn add-path-sep-suffix [^String path] (if (or (empty? path) (.endsWith path "/") @@ -339,6 +376,24 @@ (for [[i v] (map-indexed vector items)] [(str prefix i) v]))) +(defn- fname->var + "Convert a file name to a var if it exists. If it + does not exist, and is not optional, use empty + string as var representation" + [filename] + (let [file-stats (make-file-stats filename)] + (if (and (not (:exists file-stats)) + (:optional file-stats)) + "" + (:path file-stats)))) + +(defn existing-inputs-map + "Like inouts-map, except optional but nonexisting + input files are replaced by empty strings" + [items prefix] + (let [items (map fname->var items)] + (inouts-map items prefix))) + ;; TODO(artem) ;; Should we move this to a common library? (defn demix diff --git a/src/drake/parser_utils.clj b/src/drake/parser_utils.clj index e8bc27b..3ac85cc 100644 --- a/src/drake/parser_utils.clj +++ b/src/drake/parser_utils.clj @@ -240,4 +240,4 @@ (p/complex [_ single-quote chars (p/rep* (p/except p/anything single-quote)) _ single-quote] - (apply str chars))) \ No newline at end of file + (apply str chars)))