Skip to content

Commit

Permalink
Unified row-picker for scan
Browse files Browse the repository at this point in the history
  • Loading branch information
@FiV0
FiV0 committed Aug 11, 2023
1 parent f6b622c commit 35522f8
Show file tree
Hide file tree
Showing 7 changed files with 390 additions and 605 deletions.
488 changes: 123 additions & 365 deletions core/src/main/clojure/xtdb/operator/scan.clj
View file

Large diffs are not rendered by default.

20 changes: 9 additions & 11 deletions src/test/clojure/xtdb/as_of_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@
:xt/valid-from system-time
:xt/valid-to end-of-time-zdt
:xt/system-from system-time
:xt/system-to nil}
:xt/system-to (util/->zdt util/end-of-time)}
:doc-with-app-time {:xt/id :doc-with-app-time,
:xt/valid-from (util/->zdt #inst "2021")
:xt/valid-to end-of-time-zdt
:xt/system-from system-time
:xt/system-to nil}}
:xt/system-to (util/->zdt util/end-of-time)}}
(->> (tu/query-ra '[:scan {:table xt_docs}
[xt/id
xt/valid-from xt/valid-to
Expand All @@ -67,25 +67,23 @@

original-v0-doc {:xt/id :doc, :version 0
:xt/valid-from tt1
:xt/valid-to end-of-time-zdt
:xt/valid-to tt2
:xt/system-from tt1
:xt/system-to tt2}
:xt/system-to end-of-time-zdt}

replaced-v0-doc {:xt/id :doc, :version 0
:xt/valid-from tt1
:xt/valid-to tt2
:xt/system-from tt2
:xt/system-to end-of-time-zdt}
:xt/valid-from tt2
:xt/valid-to end-of-time-zdt
:xt/system-from tt1
:xt/system-to tt2}

v1-doc {:xt/id :doc, :version 1
:xt/valid-from tt2
:xt/valid-to end-of-time-zdt
:xt/system-from tt2
:xt/system-to end-of-time-zdt}]

;; TODO nil system-to #2655
(t/is (= #{(assoc replaced-v0-doc :xt/system-to nil)
(assoc v1-doc :xt/system-to nil)}
(t/is (= #{original-v0-doc v1-doc}
(set (tu/query-ra '[:scan {:table xt_docs}
[xt/id version
xt/valid-from xt/valid-to
Expand Down
27 changes: 14 additions & 13 deletions src/test/clojure/xtdb/datalog_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1504,13 +1504,13 @@
"cross-time join - who was here in both 2018 and 2023?")

(t/is (= #{{:vt-start (util/->zdt #inst "2021")
:vt-end (util/->zdt util/end-of-time)
:tt-start (util/->zdt #inst "2020-01-01")
:tt-end (util/->zdt #inst "2020-01-02")}
{:vt-start (util/->zdt #inst "2021")
:vt-end (util/->zdt #inst "2022")
:tt-start (util/->zdt #inst "2020-01-02")
:tt-end (util/->zdt util/end-of-time)}}
:tt-start (util/->zdt #inst "2020")
:tt-end (util/->zdt util/end-of-time)}
{:vt-start (util/->zdt #inst "2022")
:vt-end (util/->zdt util/end-of-time)
:tt-start (util/->zdt #inst "2020")
:tt-end (util/->zdt #inst "2020-01-02")}}
(set (q '{:find [vt-start vt-end tt-start tt-end]
:where [(match :xt_docs {:xt/id :luke
:xt/valid-from vt-start
Expand Down Expand Up @@ -1657,13 +1657,14 @@
tx1, nil)))
"as-of 18 Jan")

(t/is (= [{:cust 145, :app-start (util/->zdt #inst "1998-01-05")}
{:cust 827, :app-start (util/->zdt #inst "1998-01-12")}]
(q '{:find [cust app-start]
:where [(match :docs {:customer-number cust, :xt/valid-from app-start}
{:for-valid-time :all-time})]
:order-by [[app-start :asc]]}
tx5, nil))
(t/is (= #{{:cust 145, :app-start (util/->zdt #inst "1998-01-05")}
{:cust 827, :app-start (util/->zdt #inst "1998-01-12")}}
(set (q '{:find [cust app-start]
:where [(match :docs {:customer-number cust,
:xt/valid-from app-start}
{:for-valid-time :all-time})]
:order-by [[app-start :asc]]}
tx5, nil)))
"as-of 29 Jan")

(t/is (= [{:cust 827, :app-start (util/->zdt #inst "1998-01-12"), :app-end (util/->zdt #inst "1998-01-20")}]
Expand Down
6 changes: 3 additions & 3 deletions src/test/clojure/xtdb/indexer_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@
:xt/valid-from (util/->zdt tt)
:xt/valid-to (util/->zdt util/end-of-time)
:xt/system-from (util/->zdt tt)
:xt/system-to nil}]
:xt/system-to (util/->zdt util/end-of-time)}]
(tu/query-ra '[:scan {:table xt_docs}
[xt/id version
xt/valid-from, xt/valid-to
Expand All @@ -195,14 +195,14 @@
(let [{tt2 :system-time} (xt/submit-tx node [[:put :xt_docs {:xt/id :foo, :version 1}]]
{:default-all-valid-time? false})]
(t/is (= #{{:xt/id :foo, :version 0,
:xt/valid-from (util/->zdt tt)
:xt/valid-from (util/->zdt tt2)
:xt/valid-to (util/->zdt util/end-of-time)
:xt/system-from (util/->zdt tt)
:xt/system-to (util/->zdt tt2)}
{:xt/id :foo, :version 0,
:xt/valid-from (util/->zdt tt)
:xt/valid-to (util/->zdt tt2)
:xt/system-from (util/->zdt tt2)
:xt/system-from (util/->zdt tt)
:xt/system-to (util/->zdt util/end-of-time)}
{:xt/id :foo, :version 1,
:xt/valid-from (util/->zdt tt2)
Expand Down
227 changes: 183 additions & 44 deletions src/test/clojure/xtdb/operator/scan_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,41 +61,40 @@
[:delete :xt_docs :doc3]])]

;; valid-time
(t/is (= #{{:v 1, :xt/id :doc1} {:v 1, :xt/id :doc2}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2017"], :for-system-time nil}
[xt/id v]]
{:node node}))))

(t/is (= #{{:v 1, :xt/id :doc2} {:v 2, :xt/id :doc1}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node}))))
(t/is (= {{:v 1, :xt/id :doc1} 1 {:v 1, :xt/id :doc2} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2017"], :for-system-time nil}
[xt/id v]]
{:node node}))))
(t/is (= {{:v 1, :xt/id :doc2} 1 {:v 2, :xt/id :doc1} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node}))))
;; system-time
(t/is (= #{{:v 1, :xt/id :doc1} {:v 1, :xt/id :doc2} {:v 1, :xt/id :doc3}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx1}}))))

(t/is (= #{{:v 1, :xt/id :doc1} {:v 1, :xt/id :doc2}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2017"], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx1}}))))

(t/is (= #{{:v 2, :xt/id :doc1} {:v 1, :xt/id :doc2}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx2}}))))

(t/is (= #{{:v 2, :xt/id :doc1} {:v 2, :xt/id :doc2}}
(set (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2100"], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx2}})))))))
(t/is (= {{:v 1, :xt/id :doc1} 1 {:v 1, :xt/id :doc2} 1 {:v 1, :xt/id :doc3} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx1}}))))

(t/is (= {{:v 1, :xt/id :doc1} 1 {:v 1, :xt/id :doc2} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2017"], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx1}}))))

(t/is (= {{:v 2, :xt/id :doc1} 1 {:v 1, :xt/id :doc2} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at :now], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx2}}))))

(t/is (= {{:v 2, :xt/id :doc1} 1 {:v 2, :xt/id :doc2} 1}
(frequencies (tu/query-ra '[:scan
{:table xt_docs, :for-valid-time [:at #inst "2100"], :for-system-time nil}
[xt/id v]]
{:node node :basis {:tx tx2}})))))))

(t/deftest test-past-point-point-queries-with-valid-time
(with-open [node (node/start-node tu/*node-opts*)]
Expand Down Expand Up @@ -210,7 +209,7 @@
(t/is (= #{{:xt/valid-from (util/->zdt tt)
:xt/valid-to (util/->zdt util/end-of-time)
:xt/system-from (util/->zdt tt),
:xt/system-to nil}}
:xt/system-to (util/->zdt util/end-of-time)}}
(set (tu/query-ra '[:scan {:table xt_docs}
[xt/valid-from xt/valid-to
xt/system-from xt/system-to]]
Expand All @@ -224,18 +223,30 @@

#_(tu/finish-chunk! node)

(t/is (= #{{:xt/system-from (util/->zdt #inst "3000")
:xt/system-to (util/->zdt #inst "3001")
:last_updated "tx1"}
{:xt/system-from (util/->zdt #inst "3001")
:xt/system-to (util/->zdt util/end-of-time)
:last_updated "tx1"}
{:xt/system-from (util/->zdt #inst "3001")
:xt/system-to (util/->zdt util/end-of-time)
:last_updated "tx2"}}
(t/is (= #{{:last_updated "tx2",
:xt/valid-from #time/zoned-date-time "3001-01-01T00:00Z[UTC]",
:xt/valid-to
#time/zoned-date-time "9999-12-31T23:59:59.999999Z[UTC]",
:xt/system-from #time/zoned-date-time "3001-01-01T00:00Z[UTC]",
:xt/system-to
#time/zoned-date-time "9999-12-31T23:59:59.999999Z[UTC]"}
{:last_updated "tx1",
:xt/valid-from #time/zoned-date-time "3000-01-01T00:00Z[UTC]",
:xt/valid-to #time/zoned-date-time "3001-01-01T00:00Z[UTC]",
:xt/system-from #time/zoned-date-time "3000-01-01T00:00Z[UTC]",
:xt/system-to
#time/zoned-date-time "9999-12-31T23:59:59.999999Z[UTC]"}
{:last_updated "tx1",
:xt/valid-from #time/zoned-date-time "3001-01-01T00:00Z[UTC]",
:xt/valid-to
#time/zoned-date-time "9999-12-31T23:59:59.999999Z[UTC]",
:xt/system-from #time/zoned-date-time "3000-01-01T00:00Z[UTC]",
:xt/system-to #time/zoned-date-time "3001-01-01T00:00Z[UTC]"}}
(set (tu/query-ra '[:scan {:table foo, :for-system-time :all-time}
[{xt/system-from (< xt/system-from #time/zoned-date-time "3002-01-01T00:00Z")}
{xt/system-to (> xt/system-to #time/zoned-date-time "2999-01-01T00:00Z")}
xt/valid-from
xt/valid-to
last_updated]]
{:node node :default-all-valid-time? true}))))))

Expand Down Expand Up @@ -371,3 +382,131 @@
(tu/query-ra
'[:scan {:table xt_docs} [xt/id {col-x (= col-x "toto")}]]
{:node node})))))

(t/deftest test-correct-rectangle-cutting
(with-open [node (node/start-node {})]
(letfn [(q [id]
(-> (xt/q node {:find '[v vf vt]
:where [(list 'match :xt_docs
['v {:xt/id id :xt/valid-from 'vf :xt/valid-to 'vt}]
{:for-valid-time :all-time})]})
frequencies))]
(t/testing "period starts before and does NOT overlap"
(xt/submit-tx node [[:put :xt_docs {:xt/id 1, :v 1} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 1, :v 2} {:for-valid-time [:in #inst "2005" #inst "2009"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2005-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2009-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1}
(q 1))))

(t/testing "period starts before and overlaps"
(xt/submit-tx node [[:put :xt_docs {:xt/id 2, :v 1} {:for-valid-time [:in #inst "2015" #inst "2025"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 2, :v 2} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2020-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1}
(q 2))))

(t/testing "period starts equally and overlaps"
(xt/submit-tx node [[:put :xt_docs {:xt/id 3, :v 1} {:for-valid-time [:in #inst "2010" #inst "2025"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 3, :v 2} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2020-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1}
(q 3))))

(t/testing "newer period completely covered"
(xt/submit-tx node [[:put :xt_docs {:xt/id 4, :v 1} {:for-valid-time [:in #inst "2010" #inst "2025"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 4, :v 2} {:for-valid-time [:in #inst "2015" #inst "2020"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2015-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2015-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2020-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1}
(q 4))))

(t/testing "older period completely covered"
(xt/submit-tx node [[:put :xt_docs {:xt/id 5, :v 1} {:for-valid-time [:in #inst "2015" #inst "2020"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 5, :v 2} {:for-valid-time [:in #inst "2010" #inst "2025"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1}
(q 5))))

(t/testing "period end equally and overlaps"
(xt/submit-tx node [[:put :xt_docs {:xt/id 6, :v 1} {:for-valid-time [:in #inst "2010" #inst "2025"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 6, :v 2} {:for-valid-time [:in #inst "2015" #inst "2025"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2015-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2015-01-01T00:00Z[UTC]"} 1}
(q 6))))

(t/testing "period ends after and overlaps"
(xt/submit-tx node [[:put :xt_docs {:xt/id 7, :v 1} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 7, :v 2} {:for-valid-time [:in #inst "2015" #inst "2025"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2015-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2025-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2015-01-01T00:00Z[UTC]"} 1}
(q 7))))

(t/testing "period starts before and touches"
(xt/submit-tx node [[:put :xt_docs {:xt/id 8, :v 1} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 8, :v 2} {:for-valid-time [:in #inst "2005" #inst "2010"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2005-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2010-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1}
(q 8))))

(t/testing "period starts after and touches"
(xt/submit-tx node [[:put :xt_docs {:xt/id 9, :v 1} {:for-valid-time [:in #inst "2005" #inst "2010"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 9, :v 2} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2005-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2010-01-01T00:00Z[UTC]"} 1}
(q 9))))

(t/testing "period starts after and does NOT overlap"
(xt/submit-tx node [[:put :xt_docs {:xt/id 10, :v 1} {:for-valid-time [:in #inst "2005" #inst "2009"]}]])
(xt/submit-tx node [[:put :xt_docs {:xt/id 10, :v 2} {:for-valid-time [:in #inst "2010" #inst "2020"]}]])

(t/is (= {{:v 2,
:vf #time/zoned-date-time "2010-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2020-01-01T00:00Z[UTC]"} 1
{:v 1,
:vf #time/zoned-date-time "2005-01-01T00:00Z[UTC]",
:vt #time/zoned-date-time "2009-01-01T00:00Z[UTC]"} 1}
(q 10)))))))
3 changes: 2 additions & 1 deletion src/test/clojure/xtdb/operator_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@
(t/is (= #{"tx1" "tx2"}
(q '{xt$system_from (<= xt$system_from ?system-time2)})))

(t/is (= #{"tx1" "tx2"}
;; this test depends on how one cuts rectangles
(t/is (= #{"tx2"} #_#{"tx1" "tx2"}
(q '{xt$system_from (> xt$system_from ?system-time1)})))

(t/is (= #{}
Expand Down
Loading

0 comments on commit 35522f8

Please sign in to comment.