From ba6cfbd6c43ca0ec6642ec0248e7f673a5c36329 Mon Sep 17 00:00:00 2001 From: FiVo Date: Mon, 7 Aug 2023 10:06:47 +0200 Subject: [PATCH] Unify row-pickers into one picker --- core/src/main/clojure/xtdb/operator/scan.clj | 396 +++++------------- src/test/clojure/xtdb/as_of_test.clj | 20 +- src/test/clojure/xtdb/indexer_test.clj | 6 +- src/test/clojure/xtdb/operator/scan_test.clj | 2 +- src/test/clojure/xtdb/operator_test.clj | 3 +- .../sql/logic_test/direct-sql/sl-demo.test | 224 +++------- 6 files changed, 174 insertions(+), 477 deletions(-) diff --git a/core/src/main/clojure/xtdb/operator/scan.clj b/core/src/main/clojure/xtdb/operator/scan.clj index 95f0d3b94a..a99c467227 100644 --- a/core/src/main/clojure/xtdb/operator/scan.clj +++ b/core/src/main/clojure/xtdb/operator/scan.clj @@ -15,19 +15,18 @@ [xtdb.vector.reader :as vr] [xtdb.vector.writer :as vw] xtdb.watermark) - (:import (clojure.lang IFn IPersistentMap MapEntry) - (java.util ArrayList Arrays Iterator LinkedList List Map) + (:import (clojure.lang IPersistentMap MapEntry ) + (java.util ArrayList Arrays Iterator LinkedList List Map TreeSet) (java.util.function IntConsumer) org.apache.arrow.memory.ArrowBuf org.apache.arrow.memory.BufferAllocator [org.apache.arrow.memory.util ArrowBufPointer] - (org.apache.arrow.vector BigIntVector NullVector VectorLoader VectorSchemaRoot) - (org.apache.arrow.vector.complex ListVector StructVector) + (org.apache.arrow.vector VectorLoader VectorSchemaRoot) xtdb.api.protocols.TransactionInstant xtdb.buffer_pool.IBufferPool xtdb.ICursor xtdb.indexer.live_index.ILiveTableWatermark - (xtdb.metadata IMetadataManager ITableMetadata) + (xtdb.metadata IMetadataManager) xtdb.object_store.ObjectStore xtdb.operator.IRelationSelector (xtdb.trie ArrowHashTrie ArrowHashTrie$Leaf HashTrie LeafMerge LeafMerge$LeafPointer LiveHashTrie$Leaf) @@ -169,9 +168,6 @@ (apply-constraint for-system-time "xt$system_from" "xt$system_to"))) range)) -(defn- scan-op-at-now [scan-op] - (= :now (first (second scan-op)))) - (defn- scan-op-point? [scan-op] (= :at (first scan-op))) @@ -190,251 +186,21 @@ (scan-op-point? for-system-time)) (>= (util/instant->micros (:current-time basis)) (util/instant->micros (:system-time (:tx basis)))))) + (defn tables-with-cols [basis ^IWatermarkSource wm-src ^IScanEmitter scan-emitter] (let [{:keys [tx, after-tx]} basis wm-tx (or tx after-tx)] (with-open [^Watermark wm (.openWatermark wm-src wm-tx)] (.allTableColNames scan-emitter wm)))) -(defn- point-point-row-picker - ^java.util.function.IntConsumer [^IRelationWriter out-rel, ^RelationReader leaf-rel - col-names, ^longs temporal-range, - {:keys [^longs current-bounds skip-iid-ptr prev-iid-ptr current-iid-ptr]}] - - (let [leaf-row-count (.rowCount leaf-rel) - iid-rdr (.readerForName leaf-rel "xt$iid") - sys-from-rdr (.readerForName leaf-rel "xt$system_from") - op-rdr (.readerForName leaf-rel "op") - put-rdr (.legReader op-rdr :put) - doc-rdr (.structKeyReader put-rdr "xt$doc") - put-valid-from-rdr (.structKeyReader put-rdr "xt$valid_from") - put-valid-to-rdr (.structKeyReader put-rdr "xt$valid_to") - - delete-rdr (.legReader op-rdr :delete) - delete-valid-from-rdr (.structKeyReader delete-rdr "xt$valid_from") - delete-valid-to-rdr (.structKeyReader delete-rdr "xt$valid_to") - - row-copiers (vec - (for [col-name col-names - :let [normalized-name (util/str->normal-form-str col-name) - ^IVectorReader rdr (case normalized-name - "xt$iid" (.readerForName leaf-rel "xt$iid") - "xt$system_from" sys-from-rdr - "xt$system_to" (vr/vec->reader - (doto (NullVector. "xt$system_to") - (.setValueCount leaf-row-count))) - "xt$valid_from" nil - "xt$valid_to" nil - (.structKeyReader doc-rdr normalized-name))] - :when rdr] - (.rowCopier rdr - (case normalized-name - "xt$iid" (.writerForName out-rel col-name [:fixed-size-binary 16]) - "xt$system_from" (.writerForName out-rel col-name types/temporal-col-type) - "xt$system_to" (.writerForName out-rel col-name - [:union #{:null types/temporal-col-type}]) - (.writerForName out-rel col-name))))) - - valid-from-wtrs (vec - (for [col-name col-names - :when (= "xt$valid_from" (util/str->normal-form-str col-name))] - (.writerForName out-rel col-name types/temporal-col-type))) - - valid-to-wtrs (vec - (for [col-name col-names - :when (= "xt$valid_to" (util/str->normal-form-str col-name))] - (.writerForName out-rel col-name types/temporal-col-type))) - - valid-time (aget temporal-range valid-from-upper-idx) - system-time (aget temporal-range system-from-upper-idx)] - - (letfn [(duplicate-ptr [^ArrowBufPointer dst, ^ArrowBufPointer src] - (.set dst (.getBuf src) (.getOffset src) (.getLength src))) - - (constrain-bounds [^long valid-from, ^long valid-to] - (when (and (< valid-to valid-time) - (< (aget current-bounds 0) valid-to)) - (aset current-bounds 0 valid-to)) - - (when (and (> valid-from valid-time) - (< valid-from (aget current-bounds 1))) - (aset current-bounds 1 valid-from)))] - - (reify IntConsumer - (accept [_ idx] - (when-not (= skip-iid-ptr (.getPointer iid-rdr idx current-iid-ptr)) - (when-not (= prev-iid-ptr current-iid-ptr) - (aset current-bounds 0 Long/MIN_VALUE) - (aset current-bounds 1 Long/MAX_VALUE) - (duplicate-ptr prev-iid-ptr current-iid-ptr)) - - (if (= :evict (.getLeg op-rdr idx)) - (duplicate-ptr skip-iid-ptr current-iid-ptr) - (when (<= (.getLong sys-from-rdr idx) system-time) - (case (.getLeg op-rdr idx) - :put - (let [valid-from (.getLong put-valid-from-rdr idx) - valid-to (.getLong put-valid-to-rdr idx)] - (if (and (<= valid-from valid-time) (< valid-time valid-to)) - (do - (duplicate-ptr skip-iid-ptr current-iid-ptr) - (.startRow out-rel) - (doseq [^IRowCopier copier row-copiers] - (.copyRow copier idx)) - (doseq [^IVectorWriter valid-from-wtr valid-from-wtrs] - (.writeLong valid-from-wtr (Long/max valid-from (aget current-bounds 0)))) - (doseq [^IVectorWriter valid-to-wtr valid-to-wtrs] - (.writeLong valid-to-wtr (Long/min valid-to (aget current-bounds 1)))) - (.endRow out-rel)) - - (constrain-bounds valid-from valid-to))) - - :delete - (let [valid-from (.getLong delete-valid-from-rdr idx) - valid-to (.getLong delete-valid-to-rdr idx)] - (if (and (<= valid-from valid-time) (< valid-time valid-to)) - (duplicate-ptr skip-iid-ptr current-iid-ptr) - (constrain-bounds valid-from valid-to)))))))))))) - -(deftype Interval [^long start, ^long end, ^long sys-from]) - -(defn range-point-row-picker - ^java.util.function.IntConsumer [^IRelationWriter out-rel, ^RelationReader leaf-rel - col-names, ^longs temporal-ranges, - {:keys [^TreeSet !ranges skip-iid-ptr prev-iid-ptr current-iid-ptr]}] - (let [leaf-row-count (.rowCount leaf-rel) - iid-rdr (.readerForName leaf-rel "xt$iid") - sys-from-rdr (.readerForName leaf-rel "xt$system_from") - op-rdr (.readerForName leaf-rel "op") - put-rdr (.legReader op-rdr :put) - doc-rdr (.structKeyReader put-rdr "xt$doc") - put-valid-from-rdr (.structKeyReader put-rdr "xt$valid_from") - put-valid-to-rdr (.structKeyReader put-rdr "xt$valid_to") - - delete-rdr (.legReader op-rdr :delete) - delete-valid-from-rdr (.structKeyReader delete-rdr "xt$valid_from") - delete-valid-to-rdr (.structKeyReader delete-rdr "xt$valid_to") - - valid-from-lower (aget temporal-ranges valid-from-lower-idx) - valid-from-upper (aget temporal-ranges valid-from-upper-idx) - valid-to-lower (aget temporal-ranges valid-to-lower-idx) - valid-to-upper (aget temporal-ranges valid-to-upper-idx) - system-time (aget temporal-ranges system-to-lower-idx) - - row-copiers (vec - (for [col-name col-names - :let [normalized-name (util/str->normal-form-str col-name) - ^IVectorReader rdr (case normalized-name - "xt$iid" (.readerForName leaf-rel "xt$iid") - "xt$system_from" nil - "xt$system_to" (vr/vec->reader - (doto (NullVector. "xt$system_to") - (.setValueCount leaf-row-count))) - "xt$valid_from" nil - "xt$valid_to" nil - (.structKeyReader doc-rdr normalized-name))] - :when rdr] - (.rowCopier rdr - (case normalized-name - "xt$iid" (.writerForName out-rel col-name [:fixed-size-binary 16]) - "xt$system_to" (.writerForName out-rel col-name - [:union #{:null types/temporal-col-type}]) - (.writerForName out-rel col-name))))) - - valid-from-wtrs (vec - (for [col-name col-names - :when (= "xt$valid_from" (util/str->normal-form-str col-name))] - (.writerForName out-rel col-name types/temporal-col-type))) - - valid-to-wtrs (vec - (for [col-name col-names - :when (= "xt$valid_to" (util/str->normal-form-str col-name))] - (.writerForName out-rel col-name types/temporal-col-type))) - - sys-from-wtrs (vec - (for [col-name col-names - :when (= "xt$system_from" (util/str->normal-form-str col-name))] - (.writerForName out-rel col-name types/temporal-col-type))) - - !new-ranges (LinkedList.)] - - (letfn [(duplicate-ptr [^ArrowBufPointer dst, ^ArrowBufPointer src] - (.set dst (.getBuf src) (.getOffset src) (.getLength src)))] - - (reify IntConsumer - (accept [_ idx] - (when-not (= skip-iid-ptr (.getPointer iid-rdr idx current-iid-ptr)) - (when-not (= prev-iid-ptr current-iid-ptr) - (.clear !ranges) - (duplicate-ptr prev-iid-ptr current-iid-ptr)) - - (if (= :evict (.getLeg op-rdr idx)) - (duplicate-ptr skip-iid-ptr current-iid-ptr) - (let [sys-from (.getLong sys-from-rdr idx)] - (when (<= sys-from system-time) - (case (.getLeg op-rdr idx) - :put - (let [valid-from (.getLong put-valid-from-rdr idx) - valid-to (.getLong put-valid-to-rdr idx) - i1 (Interval. valid-from valid-to sys-from) - lower-bound (or (.lower !ranges i1) i1) - sub-range (.subSet !ranges lower-bound true (Interval. valid-to -1 -1) false)] - (if-not (.isEmpty sub-range) - (let [itr (.iterator sub-range)] - (loop [^Interval prev nil ^Interval next (.next itr)] - (let [new-valid-from (or (and prev (.end prev)) valid-from) - new-valid-to (or (and next (.start next)) valid-to)] - (when (< new-valid-from new-valid-to) - (.add !new-ranges (Interval. new-valid-from new-valid-to sys-from))) - (when next - (recur next (and (.hasNext itr) (.next itr))))))) - (.add !new-ranges i1)) - (doseq [^Interval i !new-ranges] - (.add !ranges i) - (let [valid-from (.start i) - valid-to (.end i) - sys-from (.sys-from i)] - (when (and (<= valid-from-lower valid-from) - (<= valid-from valid-from-upper) - (<= valid-to-lower valid-to) - (<= valid-to valid-to-upper)) - (.startRow out-rel) - (doseq [^IRowCopier copier row-copiers] - (.copyRow copier idx)) - (doseq [^IVectorWriter valid-from-wtr valid-from-wtrs] - (.writeLong valid-from-wtr valid-from)) - (doseq [^IVectorWriter valid-to-wtr valid-to-wtrs] - (.writeLong valid-to-wtr valid-to)) - (doseq [^IVectorWriter sys-from-wtr sys-from-wtrs] - (.writeLong sys-from-wtr sys-from)) - (.endRow out-rel)))) - (.clear !new-ranges)) - - :delete - (let [valid-from (.getLong delete-valid-from-rdr idx) - valid-to (.getLong delete-valid-to-rdr idx) - sub-range (.subSet !ranges (Interval. valid-from -1 -1) true (Interval. valid-to -1 -1) false)] - (if-not (.isEmpty sub-range) - (let [itr (.iterator sub-range)] - (loop [^Interval prev nil ^Interval next (.next itr)] - (let [new-valid-from (or (and prev (.end prev)) valid-from) - new-valid-to (or (and next (.start next)) valid-to)] - (when (< new-valid-from new-valid-to) - (.add !new-ranges (Interval. new-valid-from new-valid-to sys-from))) - (when next - (recur next (and (.hasNext itr) (.next itr))))))) - (.add !new-ranges (Interval. valid-from valid-to sys-from))) - (doseq [^Interval i !new-ranges] - (.add !ranges i)) - (.clear !new-ranges)))))))))))) - (deftype Rectangle [^long valid-from, ^long valid-to, ^long sys-from, ^long sys-to]) (defn range-range-row-picker ^java.util.function.IntConsumer [^IRelationWriter out-rel, ^RelationReader leaf-rel col-names, ^longs temporal-ranges, - {:keys [^TreeSet !ranges skip-iid-ptr prev-iid-ptr current-iid-ptr]}] + {:keys [^TreeSet !ranges skip-iid-ptr prev-iid-ptr current-iid-ptr + point-point? range-point? range-range?]}] (let [iid-rdr (.readerForName leaf-rel "xt$iid") sys-from-rdr (.readerForName leaf-rel "xt$system_from") op-rdr (.readerForName leaf-rel "op") @@ -510,11 +276,18 @@ (when (and (<= sys-from-lower system-from) (<= system-from sys-from-upper)) (case (.getLeg op-rdr idx) :put + (let [valid-from (.getLong put-valid-from-rdr idx) valid-to (.getLong put-valid-to-rdr idx) r1 (Rectangle. valid-from valid-to system-from util/end-of-time-μs) - lower-bound (or (.lower !ranges r1) r1) - sub-range (.subSet !ranges lower-bound true (Rectangle. valid-to -1 -1 -1) false)] + ^Rectangle lower-bound (.lower !ranges r1) + sub-range (.subSet !ranges + (if (and lower-bound + (<= (.valid-from lower-bound) valid-from) + (< valid-from (.valid-to lower-bound))) lower-bound r1) + true + (Rectangle. valid-to -1 -1 -1) + false)] (if-not (.isEmpty sub-range) (let [itr (.iterator sub-range)] (loop [^Rectangle prev nil ^Rectangle next (.next itr)] @@ -524,10 +297,12 @@ (.add !new-ranges (Rectangle. new-valid-from new-valid-to system-from util/end-of-time-μs))) (when next - (.add !new-ranges (Rectangle. (.valid-from next) (min valid-to (.valid-to next)) - system-from (.sys-from next))) + (when range-range? + (.add !new-ranges (Rectangle. (max valid-from (.valid-from next)) (min valid-to (.valid-to next)) + system-from (.sys-from next)))) (recur next (and (.hasNext itr) (.next itr))))))) (.add !new-ranges r1)) + (doseq [^Rectangle r !new-ranges] (let [valid-from (.valid-from r) valid-to (.valid-to r) @@ -554,24 +329,47 @@ (doseq [^IVectorWriter sys-to-wtr sys-to-wtrs] (.writeLong sys-to-wtr sys-to)) (.endRow out-rel)))) - (let [^Rectangle lower (when (not (.isEmpty sub-range)) (.first sub-range)) - ^Rectangle upper (when (not (.isEmpty sub-range)) (.last sub-range)) - itr (.iterator sub-range)] - (while (.hasNext itr) - (.next itr) - (.remove itr)) - (when (and lower (< (.valid-from lower) valid-from)) - (.add !ranges (Rectangle. (.valid-from lower) valid-from (.sys-from lower) (.sys-to lower)))) - (when (and upper (< valid-to (.valid-to upper))) - (.add !ranges (Rectangle. valid-to (.valid-to upper) (.sys-from upper) (.sys-to upper)))) - (.add !ranges r1) - (.clear !new-ranges))) + + (cond + point-point? + (doseq [^Rectangle r !new-ranges] + (.add !ranges r) + (when (and (<= (.valid-from r) valid-from-upper) (< valid-from-upper (.valid-to r))) + (duplicate-ptr skip-iid-ptr current-iid-ptr))) + + range-point? + (doseq [r !new-ranges] + (.add !ranges r)) + + range-range? + (let [^Rectangle lower (when (not (.isEmpty sub-range)) (.first sub-range)) + ^Rectangle upper (when (not (.isEmpty sub-range)) (.last sub-range)) + itr (.iterator sub-range)] + (while (.hasNext itr) + (.next itr) + (.remove itr)) + (when (and lower (< (.valid-from lower) valid-from)) + (.add !ranges (Rectangle. (.valid-from lower) valid-from (.sys-from lower) (.sys-to lower)))) + (when (and upper (< valid-to (.valid-to upper))) + (.add !ranges (Rectangle. valid-to (.valid-to upper) (.sys-from upper) (.sys-to upper)))) + (.add !ranges r1))) + (.clear !new-ranges)) + :delete (let [valid-from (.getLong delete-valid-from-rdr idx) valid-to (.getLong delete-valid-to-rdr idx) r1 (Rectangle. valid-from valid-to system-from util/end-of-time-μs) - lower-bound (or (.lower !ranges r1) r1) - sub-range (.subSet !ranges lower-bound true (Rectangle. valid-to -1 -1 -1) false)] + ^Rectangle lower-bound (.lower !ranges r1) + sub-range (.subSet !ranges + (if (and lower-bound + (<= (.valid-from lower-bound) valid-from) + (< valid-from (.valid-to lower-bound))) + lower-bound + r1) + true + (Rectangle. valid-to -1 -1 -1) + false)] + (if-not (.isEmpty sub-range) (let [itr (.iterator sub-range)] (loop [^Rectangle prev nil ^Rectangle next (.next itr)] @@ -581,26 +379,39 @@ (.add !new-ranges (Rectangle. new-valid-from new-valid-to system-from util/end-of-time-μs))) (when next - (.add !new-ranges (Rectangle. (.valid-from next) (min valid-to (.valid-to next)) - system-from (.sys-from next))) + (when range-range? + (.add !new-ranges (Rectangle. (.valid-from next) (min valid-to (.valid-to next)) + system-from (.sys-from next)))) (recur next (and (.hasNext itr) (.next itr))))))) (.add !new-ranges r1)) - (let [^Rectangle lower (when (not (.isEmpty sub-range)) (.first sub-range)) - ^Rectangle upper (when (not (.isEmpty sub-range)) (.last sub-range)) - itr (.iterator sub-range)] - (while (.hasNext itr) - (.next itr) - (.remove itr)) - (when (and lower (< (.valid-from lower) valid-from)) - (.add !ranges (Rectangle. (.valid-from lower) valid-from (.sys-from lower) (.sys-to lower)))) - (when (and upper (< valid-to (.valid-to upper))) - (.add !ranges (Rectangle. valid-to (.valid-to upper) (.sys-from upper) (.sys-to upper)))) - (.add !ranges r1) - (.clear !new-ranges))))))))))))) + + (cond + point-point? + (doseq [^Rectangle r !new-ranges] + (.add !ranges r) + (when (and (<= (.valid-from r) valid-from-upper) (< valid-from-upper (.valid-to r))) + (duplicate-ptr skip-iid-ptr current-iid-ptr))) + + range-point? + (doseq [r !new-ranges] + (.add !ranges r)) + + range-range? + (let [^Rectangle lower (when (not (.isEmpty sub-range)) (.first sub-range)) + ^Rectangle upper (when (not (.isEmpty sub-range)) (.last sub-range)) + itr (.iterator sub-range)] + (while (.hasNext itr) + (.next itr) + (.remove itr)) + (when (and lower (< (.valid-from lower) valid-from)) + (.add !ranges (Rectangle. (.valid-from lower) valid-from (.sys-from lower) (.sys-to lower)))) + (when (and upper (< valid-to (.valid-to upper))) + (.add !ranges (Rectangle. valid-to (.valid-to upper) (.sys-from upper) (.sys-to upper)))) + (.add !ranges r1))) + (.clear !new-ranges)))))))))))) (deftype TrieCursor [^BufferAllocator allocator, arrow-leaves ^Iterator merge-tasks, ^ints leaf-idxs, ^ints current-arrow-page-idxs - ^IFn row-picker col-names, ^Map col-preds, ^longs temporal-timestamps, params, ^IPersistentMap picker-state] ICursor @@ -611,7 +422,7 @@ (letfn [(rel->leaf-ptr [leaf-ordinal ^RelationReader log-rdr] (let [row-count (.rowCount log-rdr) iid-rdr (.readerForName log-rdr "xt$iid") - ^IntConsumer picker (row-picker out-rel log-rdr col-names temporal-timestamps picker-state) + ^IntConsumer picker (range-range-row-picker out-rel log-rdr col-names temporal-timestamps picker-state) is-valid-buf (ArrowBufPointer.)] @@ -721,27 +532,26 @@ live-table-wm inc)) (doto (int-array (count arrow-leaves)) (Arrays/fill -1)) - (cond (at-point-point? scan-opts) - point-point-row-picker - - (range-point-query? wm basis scan-opts) - range-point-row-picker - - :else - range-range-row-picker) - col-names col-preds temporal-range params - {:current-bounds (long-array 2) - :!ranges (if (range-point-query? wm basis scan-opts) - (TreeSet. (fn [^Interval i1 ^Interval i2] - (< (.start i1) (.start i2)))) - (TreeSet. (fn [^Rectangle i1 ^Rectangle i2] - (< (.valid-from i1) (.valid-from i2))))) - :skip-iid-ptr (ArrowBufPointer.) - :prev-iid-ptr (ArrowBufPointer.) - :current-iid-ptr (ArrowBufPointer.)}) + (merge {:current-bounds (long-array 2) + :!ranges (TreeSet. (fn [^Rectangle r1 ^Rectangle r2] + (< (.valid-from r1) (.valid-from r2)) + #_(or + (and (= (.valid-from r1) (.valid-from r2)) + (< (.sys-from r1) (.sys-from r2)))))) + :skip-iid-ptr (ArrowBufPointer.) + :prev-iid-ptr (ArrowBufPointer.) + :current-iid-ptr (ArrowBufPointer.)} + (cond (at-point-point? scan-opts) + {:point-point? true} + + (range-point-query? wm basis scan-opts) + {:range-point? true} + + :else + {:range-range? true}))) (catch Throwable t (util/close (map :leaf-buf arrow-leaves)) diff --git a/src/test/clojure/xtdb/as_of_test.clj b/src/test/clojure/xtdb/as_of_test.clj index 94bd5803e0..ee0fd667e9 100644 --- a/src/test/clojure/xtdb/as_of_test.clj +++ b/src/test/clojure/xtdb/as_of_test.clj @@ -45,12 +45,12 @@ :xt/valid-from system-time :xt/valid-to end-of-time-zdt :xt/system-from system-time - :xt/system-to nil} + :xt/system-to (util/->zdt util/end-of-time)} :doc-with-app-time {:xt/id :doc-with-app-time, :xt/valid-from (util/->zdt #inst "2021") :xt/valid-to end-of-time-zdt :xt/system-from system-time - :xt/system-to nil}} + :xt/system-to (util/->zdt util/end-of-time)}} (->> (tu/query-ra '[:scan {:table xt_docs} [xt/id xt/valid-from xt/valid-to @@ -67,15 +67,15 @@ original-v0-doc {:xt/id :doc, :version 0 :xt/valid-from tt1 - :xt/valid-to end-of-time-zdt + :xt/valid-to tt2 :xt/system-from tt1 - :xt/system-to tt2} + :xt/system-to end-of-time-zdt} replaced-v0-doc {:xt/id :doc, :version 0 - :xt/valid-from tt1 - :xt/valid-to tt2 - :xt/system-from tt2 - :xt/system-to end-of-time-zdt} + :xt/valid-from tt2 + :xt/valid-to end-of-time-zdt + :xt/system-from tt1 + :xt/system-to tt2} v1-doc {:xt/id :doc, :version 1 :xt/valid-from tt2 @@ -83,9 +83,7 @@ :xt/system-from tt2 :xt/system-to end-of-time-zdt}] - ;; TODO nil system-to #2655 - (t/is (= #{(assoc replaced-v0-doc :xt/system-to nil) - (assoc v1-doc :xt/system-to nil)} + (t/is (= #{original-v0-doc v1-doc} (set (tu/query-ra '[:scan {:table xt_docs} [xt/id version xt/valid-from xt/valid-to diff --git a/src/test/clojure/xtdb/indexer_test.clj b/src/test/clojure/xtdb/indexer_test.clj index 0663d13588..f7591ba6a1 100644 --- a/src/test/clojure/xtdb/indexer_test.clj +++ b/src/test/clojure/xtdb/indexer_test.clj @@ -180,7 +180,7 @@ :xt/valid-from (util/->zdt tt) :xt/valid-to (util/->zdt util/end-of-time) :xt/system-from (util/->zdt tt) - :xt/system-to nil}] + :xt/system-to (util/->zdt util/end-of-time)}] (tu/query-ra '[:scan {:table xt_docs} [xt/id version xt/valid-from, xt/valid-to @@ -190,14 +190,14 @@ (let [{tt2 :system-time} (xt/submit-tx node [[:put :xt_docs {:xt/id :foo, :version 1}]] {:default-all-valid-time? false})] (t/is (= #{{:xt/id :foo, :version 0, - :xt/valid-from (util/->zdt tt) + :xt/valid-from (util/->zdt tt2) :xt/valid-to (util/->zdt util/end-of-time) :xt/system-from (util/->zdt tt) :xt/system-to (util/->zdt tt2)} {:xt/id :foo, :version 0, :xt/valid-from (util/->zdt tt) :xt/valid-to (util/->zdt tt2) - :xt/system-from (util/->zdt tt2) + :xt/system-from (util/->zdt tt) :xt/system-to (util/->zdt util/end-of-time)} {:xt/id :foo, :version 1, :xt/valid-from (util/->zdt tt2) diff --git a/src/test/clojure/xtdb/operator/scan_test.clj b/src/test/clojure/xtdb/operator/scan_test.clj index a3989075c1..f402cf118b 100644 --- a/src/test/clojure/xtdb/operator/scan_test.clj +++ b/src/test/clojure/xtdb/operator/scan_test.clj @@ -210,7 +210,7 @@ (t/is (= #{{:xt/valid-from (util/->zdt tt) :xt/valid-to (util/->zdt util/end-of-time) :xt/system-from (util/->zdt tt), - :xt/system-to nil}} + :xt/system-to (util/->zdt util/end-of-time)}} (set (tu/query-ra '[:scan {:table xt_docs} [xt/valid-from xt/valid-to xt/system-from xt/system-to]] diff --git a/src/test/clojure/xtdb/operator_test.clj b/src/test/clojure/xtdb/operator_test.clj index 29ab406154..b6a81a4981 100644 --- a/src/test/clojure/xtdb/operator_test.clj +++ b/src/test/clojure/xtdb/operator_test.clj @@ -124,7 +124,8 @@ (t/is (= #{"tx1" "tx2"} (q '{xt$system_from (<= xt$system_from ?system-time2)}))) - (t/is (= #{"tx1" "tx2"} + ;; this test depends on how one cuts rectangles + (t/is (= #{"tx2"} #_#{"tx1" "tx2"} (q '{xt$system_from (> xt$system_from ?system-time1)}))) (t/is (= #{} diff --git a/src/test/resources/xtdb/sql/logic_test/direct-sql/sl-demo.test b/src/test/resources/xtdb/sql/logic_test/direct-sql/sl-demo.test index 1c2d63cfa1..5bb29a1a6e 100644 --- a/src/test/resources/xtdb/sql/logic_test/direct-sql/sl-demo.test +++ b/src/test/resources/xtdb/sql/logic_test/direct-sql/sl-demo.test @@ -22,7 +22,7 @@ SELECT * 1998-01-10T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] -NULL +9999-12-31T23:59:59.999999Z[UTC] statement ok INSERT INTO Prop_Owner (xt$id, customer_number, property_number, xt$valid_from) @@ -44,7 +44,7 @@ SELECT * 1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] -NULL +9999-12-31T23:59:59.999999Z[UTC] query IIITTTT rowsort SELECT * @@ -64,12 +64,12 @@ SELECT * 7797 1998-01-10T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -99,6 +99,7 @@ SELECT * xt$system_to) ---- + query IIITTTT rowsort SELECT * FROM Prop_Owner @@ -117,19 +118,12 @@ SELECT * 7797 1998-01-10T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -138,12 +132,12 @@ SELECT * 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -177,26 +171,12 @@ SELECT * 7797 1998-01-10T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-04T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -205,12 +185,12 @@ SELECT * 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -238,7 +218,7 @@ SELECT * 145 7797 1998-01-03T00:00Z[UTC] -1998-01-10T00:00Z[UTC] +1998-01-05T00:00Z[UTC] 2020-01-04T00:00Z[UTC] 2020-01-05T00:00Z[UTC] 1 @@ -246,40 +226,19 @@ SELECT * 7797 1998-01-05T00:00Z[UTC] 1998-01-10T00:00Z[UTC] -2020-01-05T00:00Z[UTC] -9999-12-31T23:59:59.999999Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] 2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +9999-12-31T23:59:59.999999Z[UTC] 1 145 7797 1998-01-10T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -288,12 +247,12 @@ SELECT * 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -324,7 +283,7 @@ SELECT * 145 7797 1998-01-03T00:00Z[UTC] -1998-01-10T00:00Z[UTC] +1998-01-05T00:00Z[UTC] 2020-01-04T00:00Z[UTC] 2020-01-05T00:00Z[UTC] 1 @@ -332,7 +291,7 @@ SELECT * 7797 1998-01-05T00:00Z[UTC] 1998-01-10T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-04T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 @@ -345,34 +304,20 @@ SELECT * 145 7797 1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +1998-01-12T00:00Z[UTC] +2020-01-01T00:00Z[UTC] +2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-12T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -388,12 +333,12 @@ SELECT * 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -417,7 +362,7 @@ SELECT * 1998-01-05T00:00Z[UTC] 1998-01-12T00:00Z[UTC] 2020-01-06T00:00Z[UTC] -NULL +9999-12-31T23:59:59.999999Z[UTC] statement ok DELETE @@ -442,7 +387,7 @@ SELECT * 145 7797 1998-01-03T00:00Z[UTC] -1998-01-10T00:00Z[UTC] +1998-01-05T00:00Z[UTC] 2020-01-04T00:00Z[UTC] 2020-01-05T00:00Z[UTC] 1 @@ -450,7 +395,7 @@ SELECT * 7797 1998-01-05T00:00Z[UTC] 1998-01-10T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-04T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 @@ -463,34 +408,20 @@ SELECT * 145 7797 1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +1998-01-12T00:00Z[UTC] +2020-01-01T00:00Z[UTC] +2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-12T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -500,25 +431,18 @@ SELECT * 1998-01-12T00:00Z[UTC] 1998-01-20T00:00Z[UTC] 2020-01-06T00:00Z[UTC] -2020-01-07T00:00Z[UTC] -1 -827 -7797 -1998-01-12T00:00Z[UTC] -1998-01-20T00:00Z[UTC] -2020-01-07T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 827 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -532,11 +456,8 @@ SELECT DISTINCT x.xt$system_from ---- 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] 2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] 2020-01-06T00:00Z[UTC] -2020-01-07T00:00Z[UTC] query ITT nosort SELECT * @@ -546,6 +467,7 @@ SELECT * AS x (customer_number, xt$valid_from, xt$valid_to) ---- + query ITT nosort SELECT * FROM Prop_Owner @@ -571,7 +493,6 @@ SELECT * 1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] - query ITT rowsort SELECT * FROM Prop_Owner @@ -589,7 +510,6 @@ SELECT * 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] - query ITT nosort SELECT * FROM Prop_Owner @@ -620,24 +540,9 @@ SELECT * ---- 145 2020-01-01T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -145 -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -145 -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -145 -2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] -145 -2020-01-05T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 827 2020-01-06T00:00Z[UTC] -2020-01-07T00:00Z[UTC] -827 -2020-01-07T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] query ITT nosort @@ -649,7 +554,7 @@ SELECT * ---- 827 2020-01-06T00:00Z[UTC] -NULL +9999-12-31T23:59:59.999999Z[UTC] statement ok INSERT INTO Prop_Owner (xt$id, customer_number, property_number, xt$valid_from) @@ -673,7 +578,7 @@ SELECT * 145 7797 1998-01-03T00:00Z[UTC] -1998-01-10T00:00Z[UTC] +1998-01-05T00:00Z[UTC] 2020-01-04T00:00Z[UTC] 2020-01-05T00:00Z[UTC] 1 @@ -681,7 +586,7 @@ SELECT * 7797 1998-01-05T00:00Z[UTC] 1998-01-10T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-04T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 @@ -694,34 +599,20 @@ SELECT * 145 7797 1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-02T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-03T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -1 -145 -7797 -1998-01-10T00:00Z[UTC] -1998-01-15T00:00Z[UTC] -2020-01-04T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +1998-01-12T00:00Z[UTC] +2020-01-01T00:00Z[UTC] +2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-12T00:00Z[UTC] 1998-01-15T00:00Z[UTC] -2020-01-05T00:00Z[UTC] +2020-01-01T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 145 7797 -1998-01-10T00:00Z[UTC] +1998-01-15T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-01T00:00Z[UTC] 2020-01-02T00:00Z[UTC] @@ -731,25 +622,18 @@ SELECT * 1998-01-12T00:00Z[UTC] 1998-01-20T00:00Z[UTC] 2020-01-06T00:00Z[UTC] -2020-01-07T00:00Z[UTC] -1 -827 -7797 -1998-01-12T00:00Z[UTC] -1998-01-20T00:00Z[UTC] -2020-01-07T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 1 827 7797 1998-01-15T00:00Z[UTC] 1998-01-20T00:00Z[UTC] -2020-01-03T00:00Z[UTC] +2020-01-02T00:00Z[UTC] 2020-01-06T00:00Z[UTC] 1 827 7797 -1998-01-15T00:00Z[UTC] +1998-01-20T00:00Z[UTC] 9999-12-31T23:59:59.999999Z[UTC] 2020-01-02T00:00Z[UTC] 2020-01-03T00:00Z[UTC] @@ -770,6 +654,7 @@ WHERE P1.property_number = 7797 AND P1.customer_number = P2.customer_number ---- + query ITT nosort SELECT P2.property_number, GREATEST(P1.xt$valid_from, P2.xt$valid_from) AS VT_Begin, @@ -808,6 +693,7 @@ WHERE P1.property_number = 7797 AND P1.SYSTEM_TIME OVERLAPS P2.SYSTEM_TIME ---- + query ITTTT nosort SELECT P2.property_number, GREATEST(P1.xt$valid_from, P2.xt$valid_from) AS VT_Begin, @@ -860,6 +746,7 @@ WHERE P1.property_number = 7797 AND P1.SYSTEM_TIME CONTAINS PERIOD(P2.xt$system_from, P2.xt$system_to) ---- + query ITTT nosort SELECT P2.property_number, GREATEST(P1.xt$valid_from, P2.xt$valid_from) AS VT_Begin, @@ -889,3 +776,4 @@ SELECT P2.property_number, P2.xt$system_from AS Recorded_Start ---- 3621 2020-01-08T00:00Z[UTC] +