Make log-limit and page-limit configurable for tests

FiV0 · Aug 21, 2023 · 2b16323 · 2b16323
1 parent 60701d3
commit 2b16323
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 30 deletions.
diff --git a/core/src/main/clojure/xtdb/indexer/live_index.clj b/core/src/main/clojure/xtdb/indexer/live_index.clj
@@ -217,13 +217,20 @@
  (.structKeyWriter put-wtr "xt$doc") delete-wtr (.structKeyWriter delete-wtr "xt$valid_from")
  (.structKeyWriter delete-wtr "xt$valid_to") (.writerForField op-wtr trie/evict-field))))))
 
-(defrecord LiveIndex [^BufferAllocator allocator, ^ObjectStore object-store, ^Map tables]
+(defn ->live-trie [log-limit page-limit iid-vec]
+ (-> (doto (LiveHashTrie/builder iid-vec)
+ (.setLogLimit log-limit)
+ (.setPageLimit page-limit))
+ (.build)))
+
+(defrecord LiveIndex [^BufferAllocator allocator, ^ObjectStore object-store, ^Map tables, ^long log-limit, ^long page-limit]
  ILiveIndex
  (liveTable [_ table-name]
  (.computeIfAbsent tables table-name
  (reify Function
  (apply [_ table-name]
- (->live-table allocator object-store table-name)))))
+ (->live-table allocator object-store table-name
+ {:->live-trie (partial ->live-trie log-limit page-limit)})))))
 
  (startTx [live-idx tx-key]
  (let [table-txs (HashMap.)]
@@ -296,8 +303,9 @@
  :object-store (ig/ref :xtdb/object-store)}
  opts))
 
-(defmethod ig/init-key :xtdb.indexer/live-index [_ {:keys [allocator object-store]}]
- (->LiveIndex allocator object-store (HashMap.)))
+(defmethod ig/init-key :xtdb.indexer/live-index [_ {:keys [allocator object-store log-limit page-limit]
+ :or {log-limit 64 page-limit 1024}}]
+ (->LiveIndex allocator object-store (HashMap.) log-limit page-limit))
 
 (defmethod ig/halt-key! :xtdb.indexer/live-index [_ live-idx]
  (util/close live-idx))
diff --git a/src/main/clojure/xtdb/test_util.clj b/src/main/clojure/xtdb/test_util.clj
@@ -235,7 +235,7 @@
  (t/is (= blocks (<-cursor cursor))))))))
 
 (defn ->local-node ^java.lang.AutoCloseable [{:keys [^Path node-dir ^String buffers-dir
- rows-per-block rows-per-chunk instant-src]
+ rows-per-chunk log-limit page-limit instant-src]
  :or {buffers-dir "buffers"}}]
  (let [instant-src (or instant-src (->mock-clock))]
  (node/start-node {:xtdb.log/local-directory-log {:root-path (.resolve node-dir "log")
@@ -244,7 +244,9 @@
  :xtdb.buffer-pool/buffer-pool {:cache-path (.resolve node-dir buffers-dir)}
  :xtdb.object-store/file-system-object-store {:root-path (.resolve node-dir "objects")}
  :xtdb/indexer (->> {:rows-per-chunk rows-per-chunk}
- (into {} (filter val)))})))
+ (into {} (filter val)))
+ :xtdb.indexer/live-index (->> {:log-limit log-limit :page-limit page-limit}
+ (into {} (filter val)))})))
 
 (defn ->local-submit-node ^java.lang.AutoCloseable [{:keys [^Path node-dir]}]
  (node/start-submit-node {:xtdb.tx-producer/tx-producer {:clock (->mock-clock)}

diff --git a/src/test/clojure/xtdb/indexer/live_table_test.clj b/src/test/clojure/xtdb/indexer/live_table_test.clj
@@ -14,12 +14,6 @@
  xtdb.vector.IVectorPosition
  (xtdb.trie LiveHashTrie LiveHashTrie$Leaf)))
 
-(defn ->live-trie [log-limit page-limit iid-vec]
- (-> (doto (LiveHashTrie/builder iid-vec)
- (.setLogLimit log-limit)
- (.setPageLimit page-limit))
- (.build)))
-
 (defn uuid-equal-to-path? [uuid path]
  (Arrays/equals
  (util/uuid->bytes uuid)
@@ -36,15 +30,15 @@
  (with-open [obj-store (obj-store-test/fs path)
  allocator (RootAllocator.)
  live-table ^ILiveTable (live-index/->live-table
-  allocator obj-store "foo" {:->live-trie (partial ->live-trie 2 4)})
+ allocator obj-store "foo" {:->live-trie (partial live-index/->live-trie 2 4)})
  live-table-tx (.startTx
-  live-table (xtp/->TransactionInstant 0 (.toInstant #inst "2000")))]
+ live-table (xtp/->TransactionInstant 0 (.toInstant #inst "2000")))]
 
  (let [wp (IVectorPosition/build)]
  (dotimes [_n n]
  (.logPut
-  live-table-tx (ByteBuffer/wrap (util/uuid->bytes uuid))
-  0 0 #(.getPositionAndIncrement wp))))
+ live-table-tx (ByteBuffer/wrap (util/uuid->bytes uuid))
+ 0 0 #(.getPositionAndIncrement wp))))
 
  (.commit live-table-tx)
 
@@ -54,9 +48,9 @@
  (t/is (= 1 (count leaves)))
 
  (t/is
-  (uuid-equal-to-path?
-  uuid
-  (.path leaf)))
+ (uuid-equal-to-path?
+ uuid
+ (.path leaf)))
 
  (t/is (= (reverse (range n))
  (->> leaf
@@ -67,24 +61,24 @@
 
  (tu/with-allocator
  #(tj/check-json
-  (.toPath (io/as-file (io/resource "xtdb/live-table-test/max-depth-trie-s")))
-  path)))))
+ (.toPath (io/as-file (io/resource "xtdb/live-table-test/max-depth-trie-s")))
+ path)))))
 
  (let [uuid #uuid "7fffffff-ffff-ffff-7fff-ffffffffffff"
  n 50000]
  (tu/with-tmp-dirs #{path}
  (with-open [obj-store (obj-store-test/fs path)
  allocator (RootAllocator.)
  live-table ^ILiveTable (live-index/->live-table
-  allocator obj-store "foo")
+ allocator obj-store "foo")
  live-table-tx (.startTx
-  live-table (xtp/->TransactionInstant 0 (.toInstant #inst "2000")))]
+ live-table (xtp/->TransactionInstant 0 (.toInstant #inst "2000")))]
 
  (let [wp (IVectorPosition/build)]
  (dotimes [_n n]
  (.logPut
-  live-table-tx (ByteBuffer/wrap (util/uuid->bytes uuid))
-  0 0 #(.getPositionAndIncrement wp))))
+ live-table-tx (ByteBuffer/wrap (util/uuid->bytes uuid))
+ 0 0 #(.getPositionAndIncrement wp))))
 
  (.commit live-table-tx)
 
@@ -94,9 +88,9 @@
  (t/is (= 1 (count leaves)))
 
  (t/is
-  (uuid-equal-to-path?
-  uuid
-  (.path leaf)))
+ (uuid-equal-to-path?
+ uuid
+ (.path leaf)))
 
  (t/is (= (reverse (range n))
  (->> leaf
@@ -107,5 +101,5 @@
 
  (tu/with-allocator
  #(tj/check-json
-  (.toPath (io/as-file (io/resource "xtdb/live-table-test/max-depth-trie-l")))
-  path)))))))
+ (.toPath (io/as-file (io/resource "xtdb/live-table-test/max-depth-trie-l")))
+ path)))))))