range-point and range-range cursors for scan

FiV0 · Aug 4, 2023 · 757075f · 757075f
1 parent 49b0c54
commit 757075f
Show file tree

Hide file tree

Showing 15 changed files with 895 additions and 401 deletions.
diff --git a/core/src/main/clojure/xtdb/operator/scan.clj b/core/src/main/clojure/xtdb/operator/scan.clj
diff --git a/src/test/clojure/xtdb/api_test.clj b/src/test/clojure/xtdb/api_test.clj
@@ -330,11 +330,11 @@
  "SELECT foo.version, foo.xt$valid_from, foo.xt$valid_to FROM foo"))
  "without flag it returns as of now")
 
- (t/is (= [{:version 0, :xt$valid_from tt1, :xt$valid_to tt2}
- {:version 1, :xt$valid_from tt2, :xt$valid_to eot}]
- (xt/q *node*
- "SELECT foo.version, foo.xt$valid_from, foo.xt$valid_to FROM foo"
- {:default-all-valid-time? true})))
+ (t/is (= #{{:version 0, :xt$valid_from tt1, :xt$valid_to tt2}
+  {:version 1, :xt$valid_from tt2, :xt$valid_to eot}}
+ (set (xt/q *node*
+  "SELECT foo.version, foo.xt$valid_from, foo.xt$valid_to FROM foo"
+  {:default-all-valid-time? true}))))
 
  (t/is (= [{:version 0, :xt$valid_from tt1, :xt$valid_to tt2}]
  (xt/q *node*
@@ -344,12 +344,12 @@
  {:default-all-valid-time? true}))
  "`FOR VALID_TIME AS OF` overrides flag")
 
- (t/is (= [{:version 0, :xt$valid_from tt1, :xt$valid_to tt2}
- {:version 1, :xt$valid_from tt2, :xt$valid_to eot}]
- (xt/q *node*
- "SELECT foo.version, foo.xt$valid_from, foo.xt$valid_to
+ (t/is (= #{{:version 0, :xt$valid_from tt1, :xt$valid_to tt2}
+  {:version 1, :xt$valid_from tt2, :xt$valid_to eot}}
+ (set (xt/q *node*
+  "SELECT foo.version, foo.xt$valid_from, foo.xt$valid_to
  FROM foo FOR ALL VALID_TIME"
- {:default-all-valid-time? false}))
+  {:default-all-valid-time? false})))
  "FOR ALL VALID_TIME ignores flag and returns all app-time")))
 
 (t/deftest test-erase

diff --git a/src/test/clojure/xtdb/as_of_test.clj b/src/test/clojure/xtdb/as_of_test.clj
@@ -63,7 +63,6 @@
  tt1 (util/->zdt (:system-time tx1))
 
  tx2 (xt/submit-tx tu/*node* [[:put :xt_docs {:xt/id :doc, :version 1}]])
-
  tt2 (util/->zdt (:system-time tx2))
 
  original-v0-doc {:xt/id :doc, :version 0
@@ -84,22 +83,24 @@
  :xt/system-from tt2
  :xt/system-to end-of-time-zdt}]
 
- (t/is (= [replaced-v0-doc v1-doc]
- (tu/query-ra '[:scan {:table xt_docs}
- [xt/id version
- xt/valid-from xt/valid-to
- xt/system-from xt/system-to]]
- {:node tu/*node* :default-all-valid-time? true}))
+ ;; TODO nil system-to #2655
+ (t/is (= #{(assoc replaced-v0-doc :xt/system-to nil)
+ (assoc v1-doc :xt/system-to nil)}
+ (set (tu/query-ra '[:scan {:table xt_docs}
+ [xt/id version
+ xt/valid-from xt/valid-to
+ xt/system-from xt/system-to]]
+ {:node tu/*node* :default-all-valid-time? true})))
  "all app-time")
 
- (t/is (= [original-v0-doc replaced-v0-doc v1-doc]
- (tu/query-ra '[:scan {:table xt_docs, :for-system-time :all-time}
- [xt/id version
- xt/valid-from xt/valid-to
- xt/system-from xt/system-to]]
- {:node tu/*node*
- :params {'eot util/end-of-time}
- :default-all-valid-time? true}))
+ (t/is (= #{original-v0-doc replaced-v0-doc v1-doc}
+ (set (tu/query-ra '[:scan {:table xt_docs, :for-system-time :all-time}
+  [xt/id version
+  xt/valid-from xt/valid-to
+  xt/system-from xt/system-to]]
+  {:node tu/*node*
+  :params {'eot util/end-of-time}
+  :default-all-valid-time? true})))
  "all app, all sys")))
 
 (t/deftest test-evict

diff --git a/src/test/clojure/xtdb/datalog_test.clj b/src/test/clojure/xtdb/datalog_test.clj
@@ -10,8 +10,7 @@
  [xtdb.james-bond :as bond]
  [xtdb.node :as node]
  [xtdb.test-util :as tu]
- [xtdb.util :as util]
- [xtdb.operator.scan :as scan]))
+ [xtdb.util :as util]))
 
 (t/use-fixtures :each tu/with-mock-clock tu/with-node)
 
@@ -1474,25 +1473,25 @@
  (set (q '{:find [id], :where [(match :xt_docs [{:xt/id id}])]}, tx0, #inst "2023")))
  "back in system-time")
 
- (t/is (= [{:id :matthew, :app-start (util/->zdt #inst "2015"), :app-end (util/->zdt util/end-of-time)}
- {:id :mark, :app-start (util/->zdt #inst "2018"), :app-end (util/->zdt #inst "2020")}
- {:id :luke, :app-start (util/->zdt #inst "2021"), :app-end (util/->zdt #inst "2022")}
- {:id :mark, :app-start (util/->zdt #inst "2023"), :app-end (util/->zdt #inst "2024")}
- {:id :john, :app-start (util/->zdt #inst "2016"), :app-end (util/->zdt #inst "2020")}]
- (q '{:find [id app-start app-end]
- :where [(match :xt_docs [{:xt/id id} {:xt/valid-from app-start
- :xt/valid-to app-end}]
- {:for-valid-time :all-time})]}
- tx1, nil))
+ (t/is (= #{{:id :matthew, :app-start (util/->zdt #inst "2015"), :app-end (util/->zdt util/end-of-time)}
+  {:id :mark, :app-start (util/->zdt #inst "2018"), :app-end (util/->zdt #inst "2020")}
+  {:id :luke, :app-start (util/->zdt #inst "2021"), :app-end (util/->zdt #inst "2022")}
+  {:id :mark, :app-start (util/->zdt #inst "2023"), :app-end (util/->zdt #inst "2024")}
+  {:id :john, :app-start (util/->zdt #inst "2016"), :app-end (util/->zdt #inst "2020")}}
+ (set (q '{:find [id app-start app-end]
+  :where [(match :xt_docs [{:xt/id id} {:xt/valid-from app-start
+  :xt/valid-to app-end}]
+  {:for-valid-time :all-time})]}
+  tx1, nil)))
  "entity history, all time")
 
- (t/is (= [{:id :matthew, :app-start (util/->zdt #inst "2015"), :app-end (util/->zdt util/end-of-time)}
- {:id :luke, :app-start (util/->zdt #inst "2021"), :app-end (util/->zdt #inst "2022")}]
- (q '{:find [id app-start app-end]
- :where [(match :xt_docs [{:xt/id id} {:xt/valid-from app-start
- :xt/valid-to app-end}]
- {:for-valid-time [:in #inst "2021", #inst "2023"]})]}
- tx1, nil))
+ (t/is (= #{{:id :matthew, :app-start (util/->zdt #inst "2015"), :app-end (util/->zdt util/end-of-time)}
+  {:id :luke, :app-start (util/->zdt #inst "2021"), :app-end (util/->zdt #inst "2022")}}
+ (set (q '{:find [id app-start app-end]
+  :where [(match :xt_docs [{:xt/id id} {:xt/valid-from app-start
+  :xt/valid-to app-end}]
+  {:for-valid-time [:in #inst "2021", #inst "2023"]})]}
+  tx1, nil)))
  "entity history, range")
 
  (t/is (= #{{:id :matthew}, {:id :mark}}
@@ -1504,41 +1503,41 @@
  tx1, nil)))
  "cross-time join - who was here in both 2018 and 2023?")
 
- (t/is (= [{:vt-start (util/->zdt #inst "2021")
- :vt-end (util/->zdt util/end-of-time)
- :tt-start (util/->zdt #inst "2020-01-01")
- :tt-end (util/->zdt #inst "2020-01-02")}
- {:vt-start (util/->zdt #inst "2021")
- :vt-end (util/->zdt #inst "2022")
- :tt-start (util/->zdt #inst "2020-01-02")
- :tt-end (util/->zdt util/end-of-time)}]
- (q '{:find [vt-start vt-end tt-start tt-end]
- :where [(match :xt_docs {:xt/id :luke
- :xt/valid-from vt-start
- :xt/valid-to vt-end
- :xt/system-from tt-start
- :xt/system-to tt-end}
- {:for-valid-time :all-time
- :for-system-time :all-time})]}
- tx1 nil))
+ (t/is (= #{{:vt-start (util/->zdt #inst "2021")
+  :vt-end (util/->zdt util/end-of-time)
+  :tt-start (util/->zdt #inst "2020-01-01")
+  :tt-end (util/->zdt #inst "2020-01-02")}
+  {:vt-start (util/->zdt #inst "2021")
+  :vt-end (util/->zdt #inst "2022")
+  :tt-start (util/->zdt #inst "2020-01-02")
+  :tt-end (util/->zdt util/end-of-time)}}
+ (set (q '{:find [vt-start vt-end tt-start tt-end]
+  :where [(match :xt_docs {:xt/id :luke
+  :xt/valid-from vt-start
+  :xt/valid-to vt-end
+  :xt/system-from tt-start
+  :xt/system-to tt-end}
+  {:for-valid-time :all-time
+  :for-system-time :all-time})]}
+  tx1 nil)))
 
  "for all sys time"))))
 
 (t/deftest test-for-valid-time-with-current-time-2493
  (xt/submit-tx tu/*node* '[[:put :xt_docs {:xt/id :matthew} {:for-valid-time [:in nil #inst "2040"]}]])
  (xt/submit-tx tu/*node* '[[:put :xt_docs {:xt/id :matthew} {:for-valid-time [:in #inst "2022" #inst "2030"]}]])
- (t/is (= [{:id :matthew,
- :vt-start #time/zoned-date-time "2030-01-01T00:00Z[UTC]",
- :vt-end #time/zoned-date-time "2040-01-01T00:00Z[UTC]"}
- {:id :matthew,
- :vt-start #time/zoned-date-time "2022-01-01T00:00Z[UTC]",
- :vt-end #time/zoned-date-time "2030-01-01T00:00Z[UTC]"}]
- (xt/q tu/*node*
- '{:find [id vt-start vt-end], :where [(match :xt_docs {:xt/id id
- :xt/valid-from vt-start
- :xt/valid-to vt-end}
- {:for-valid-time [:in nil #inst "2040"]})]}
- {:basis {:current-time (util/->instant #inst "2023")}}))))
+ (t/is (= #{{:id :matthew,
+  :vt-start #time/zoned-date-time "2030-01-01T00:00Z[UTC]",
+  :vt-end #time/zoned-date-time "2040-01-01T00:00Z[UTC]"}
+  {:id :matthew,
+  :vt-start #time/zoned-date-time "2022-01-01T00:00Z[UTC]",
+  :vt-end #time/zoned-date-time "2030-01-01T00:00Z[UTC]"}}
+ (set (xt/q tu/*node*
+  '{:find [id vt-start vt-end], :where [(match :xt_docs {:xt/id id
+  :xt/valid-from vt-start
+  :xt/valid-to vt-end}
+  {:for-valid-time [:in nil #inst "2040"]})]}
+  {:basis {:current-time (util/->instant #inst "2023")}})))))
 
 (t/deftest test-temporal-opts-from-and-to
  (letfn [(q [query tx current-time]
@@ -1650,12 +1649,12 @@
  tx0, nil))
  "as-of 14 Jan")
 
- (t/is (= [{:cust 145, :app-start (util/->zdt #inst "1998-01-10")}
- {:cust 827, :app-start (util/->zdt #inst "1998-01-15")}]
- (q '{:find [cust app-start]
- :where [(match :docs {:customer-number cust, :xt/valid-from app-start}
- {:for-valid-time :all-time})]}
- tx1, nil))
+ (t/is (= #{{:cust 145, :app-start (util/->zdt #inst "1998-01-10")}
+  {:cust 827, :app-start (util/->zdt #inst "1998-01-15")}}
+ (set (q '{:find [cust app-start]
+  :where [(match :docs {:customer-number cust, :xt/valid-from app-start}
+  {:for-valid-time :all-time})]}
+  tx1, nil)))
  "as-of 18 Jan")
 
  (t/is (= [{:cust 145, :app-start (util/->zdt #inst "1998-01-05")}
@@ -2169,11 +2168,11 @@
  (xt/submit-tx tu/*node* [[:put :xt_docs {:xt/id 1 :foo "2000-4000"} {:for-valid-time [:in #inst "2000" #inst "4000"]}]
  [:put :xt_docs {:xt/id 1 :foo "3000-"} {:for-valid-time [:from #inst "3000"]}]])
 
- (t/is (= [{:xt/id 1, :foo "2000-4000"} {:xt/id 1, :foo "3000-"}]
- (xt/q tu/*node*
- '{:find [xt/id foo]
- :where [(match :xt_docs [xt/id foo])]}
- {:default-all-valid-time? true}))))
+ (t/is (= #{{:xt/id 1, :foo "2000-4000"} {:xt/id 1, :foo "3000-"}}
+ (set (xt/q tu/*node*
+  '{:find [xt/id foo]
+  :where [(match :xt_docs [xt/id foo])]}
+  {:default-all-valid-time? true})))))
 
 (t/deftest test-sql-insert
  (xt/submit-tx tu/*node* [[:sql "INSERT INTO foo (xt$id) VALUES (0)"]])
@@ -2326,25 +2325,41 @@
  (doseq [i (range 10)]
  (xt/submit-tx node [[:put :ints {:xt/id 0 :n i}]]))
  (t/is (=
- [{:n 0,
- :valid-time
- {:start #time/zoned-date-time "2020-01-01T00:00Z[UTC]",
- :end #time/zoned-date-time "2020-01-02T00:00Z[UTC]"}}
- {:n 1,
- :valid-time
- {:start #time/zoned-date-time "2020-01-02T00:00Z[UTC]",
- :end #time/zoned-date-time "2020-01-03T00:00Z[UTC]"}}
- {:n 2,
- :valid-time
- {:start #time/zoned-date-time "2020-01-03T00:00Z[UTC]",
- :end #time/zoned-date-time "2020-01-04T00:00Z[UTC]"}}
- {:n 3,
- :valid-time
- {:start #time/zoned-date-time "2020-01-04T00:00Z[UTC]",
- :end #time/zoned-date-time "2020-01-05T00:00Z[UTC]"}}
- {:n 4,
- :valid-time
- {:start #time/zoned-date-time "2020-01-05T00:00Z[UTC]",
- :end #time/zoned-date-time "2020-01-06T00:00Z[UTC]"}}]
- (xt/q node '{:find [n valid-time] :where [($ :ints {:n n :xt/id 0 :xt/valid-time valid-time}
- {:for-valid-time [:in #inst "2020-01-01" #inst "2020-01-06"]})]})))))
+ #{{:n 0,
+ :valid-time
+ {:start #time/zoned-date-time "2020-01-01T00:00Z[UTC]",
+ :end #time/zoned-date-time "2020-01-02T00:00Z[UTC]"}}
+ {:n 1,
+ :valid-time
+ {:start #time/zoned-date-time "2020-01-02T00:00Z[UTC]",
+ :end #time/zoned-date-time "2020-01-03T00:00Z[UTC]"}}
+ {:n 2,
+ :valid-time
+ {:start #time/zoned-date-time "2020-01-03T00:00Z[UTC]",
+ :end #time/zoned-date-time "2020-01-04T00:00Z[UTC]"}}
+ {:n 3,
+ :valid-time
+ {:start #time/zoned-date-time "2020-01-04T00:00Z[UTC]",
+ :end #time/zoned-date-time "2020-01-05T00:00Z[UTC]"}}
+ {:n 4,
+ :valid-time
+ {:start #time/zoned-date-time "2020-01-05T00:00Z[UTC]",
+ :end #time/zoned-date-time "2020-01-06T00:00Z[UTC]"}}}
+ (set (xt/q node '{:find [n valid-time] :where [($ :ints {:n n :xt/id 0 :xt/valid-time valid-time}
+ {:for-valid-time [:in #inst "2020-01-01" #inst "2020-01-06"]})]}))))))
+
+(deftest test-no-zero-width-intervals
+ (xt/submit-tx tu/*node* [[:put :xt-docs {:xt/id 1 :v 1}]
+ [:put :xt-docs {:xt/id 1 :v 2}]
+ [:put :xt-docs {:xt/id 2 :v 1} {:for-valid-time [:in #inst "2020-01-01" #inst "2020-01-02"]}]])
+ (xt/submit-tx tu/*node* [[:put :xt-docs {:xt/id 2 :v 2} {:for-valid-time [:in #inst "2020-01-01" #inst "2020-01-02"]}]])
+ (t/is (= [{:v 2}]
+ (xt/q tu/*node*
+ '{:find [v]
+ :where [(match :xt-docs [{:xt/id 1} v] {:for-system-time :all-time})]}))
+ "no zero width system time intervals")
+ (t/is (= [{:v 2}]
+ (xt/q tu/*node*
+ '{:find [v]
+ :where [(match :xt-docs [{:xt/id 2} v] {:for-valid-time :all-time})]}))
+ "no zero width valid-time intervals"))
diff --git a/src/test/clojure/xtdb/indexer_test.clj b/src/test/clojure/xtdb/indexer_test.clj
@@ -209,26 +209,26 @@
 
  (let [{tt2 :system-time} (xt/submit-tx node [[:put :xt_docs {:xt/id :foo, :version 1}]]
  {:default-all-valid-time? false})]
- (t/is (= [{:xt/id :foo, :version 0,
- :xt/valid-from (util/->zdt tt)
- :xt/valid-to (util/->zdt util/end-of-time)
- :xt/system-from (util/->zdt tt)
- :xt/system-to (util/->zdt tt2)}
- {:xt/id :foo, :version 0,
- :xt/valid-from (util/->zdt tt)
- :xt/valid-to (util/->zdt tt2)
- :xt/system-from (util/->zdt tt2)
- :xt/system-to (util/->zdt util/end-of-time)}
- {:xt/id :foo, :version 1,
- :xt/valid-from (util/->zdt tt2)
- :xt/valid-to (util/->zdt util/end-of-time)
- :xt/system-from (util/->zdt tt2)
- :xt/system-to (util/->zdt util/end-of-time)}]
- (tu/query-ra '[:scan {:table xt_docs, :for-system-time :all-time}
- [xt/id version
- xt/valid-from, xt/valid-to
- xt/system-from, xt/system-to]]
- {:node node :default-all-valid-time? true})))
+ (t/is (= #{{:xt/id :foo, :version 0,
+  :xt/valid-from (util/->zdt tt)
+  :xt/valid-to (util/->zdt util/end-of-time)
+  :xt/system-from (util/->zdt tt)
+  :xt/system-to (util/->zdt tt2)}
+  {:xt/id :foo, :version 0,
+  :xt/valid-from (util/->zdt tt)
+  :xt/valid-to (util/->zdt tt2)
+  :xt/system-from (util/->zdt tt2)
+  :xt/system-to (util/->zdt util/end-of-time)}
+  {:xt/id :foo, :version 1,
+  :xt/valid-from (util/->zdt tt2)
+  :xt/valid-to (util/->zdt util/end-of-time)
+  :xt/system-from (util/->zdt tt2)
+  :xt/system-to (util/->zdt util/end-of-time)}}
+ (set (tu/query-ra '[:scan {:table xt_docs, :for-system-time :all-time}
+  [xt/id version
+  xt/valid-from, xt/valid-to
+  xt/system-from, xt/system-to]]
+  {:node node :default-all-valid-time? true}))))
 
  #_ ; FIXME #567 this sees the updated xt/system-to of the first entry
  (t/is (= [{:xt/id :foo, :version 0,