add metadata to log trie files, xtdb#2650

core/src/main/clojure/xtdb/indexer/live_index.clj

@@ -1,7 +1,6 @@
 (ns xtdb.indexer.live-index
  (:require [juxt.clojars-mirrors.integrant.core :as ig]
  [xtdb.buffer-pool]
- [xtdb.metadata :as meta]
  [xtdb.object-store]
  [xtdb.trie :as trie]
  [xtdb.types :as types]
@@ -10,7 +9,6 @@
  [xtdb.vector.writer :as vw])
  (:import [clojure.lang MapEntry]
  (java.lang AutoCloseable)
- (java.nio ByteBuffer)
  (java.util ArrayList HashMap Map)
  (java.util.concurrent CompletableFuture)
  (java.util.function Function)

core/src/main/clojure/xtdb/trie.clj

@@ -3,7 +3,8 @@
  [xtdb.object-store]
  [xtdb.types :as types]
  [xtdb.util :as util]
- [xtdb.vector.writer :as vw])
+ [xtdb.vector.writer :as vw]
+ [xtdb.metadata :as meta])
  (:import (java.nio ByteBuffer)
  (java.util Arrays)
  (java.util.concurrent.atomic AtomicInteger)
@@ -21,8 +22,8 @@
  (Schema. [(types/->field "nodes" (ArrowType$Union. UnionMode/Dense (int-array (range 3))) false
  (types/col-type->field "nil" :null)
  (types/col-type->field "branch" [:list [:union #{:null :i32}]])
- ;; TODO metadata
- (types/col-type->field "leaf" '[:struct {page-idx :i32}]))]))
+ (types/col-type->field "leaf" [:struct {'page-idx :i32
+  'columns meta/metadata-col-type}]))]))
 
 (def put-field
  (types/col-type->field "put" [:struct {'xt$valid_from types/temporal-col-type
@@ -63,6 +64,7 @@
 
  leaf-wtr (.writerForTypeId node-wtr (byte 2))
  page-idx-wtr (.structKeyWriter leaf-wtr "page-idx")
+ page-meta-wtr (meta/->page-meta-wtr (.structKeyWriter leaf-wtr "columns"))
  !page-idx (AtomicInteger. 0)
  copier (vw/->rel-copier leaf-rel-wtr leaf-rel)
 
@@ -94,6 +96,21 @@
  (.copyRow copier idx)))))
 
  (.syncRowCount leaf-rel-wtr)
+
+ (let [leaf-rdr (vw/rel-wtr->rdr leaf-rel-wtr)
+ put-rdr (-> leaf-rdr
+ (.readerForName "op")
+ (.legReader :put)
+ (.metadataReader))
+
+ doc-rdr (.structKeyReader put-rdr "xt$doc")]
+
+ (.writeMetadata page-meta-wtr (into [(.readerForName leaf-rdr "xt$system_from")
+ (.readerForName leaf-rdr "xt$iid")
+ (.structKeyReader put-rdr "xt$valid_from")
+ (.structKeyReader put-rdr "xt$valid_to")]
+ (map #(.structKeyReader doc-rdr %))
+ (.structKeys doc-rdr))))
  (write-batch!)
  (.clear leaf-rel-wtr)
  (.clear leaf-vsr)
@@ -105,6 +122,7 @@
 
  (write-node! (.rootNode trie)))))]
 
+ (.syncSchema trie-vsr)
  (.syncRowCount trie-rel-wtr)
 
  {:leaf-buf leaf-buf

core/src/main/java/xtdb/vector/IVectorReader.java

@@ -82,7 +82,7 @@ default IVectorReader select(int startIdx, int len) {
  IMonoVectorReader monoReader(Object colType);
  IPolyVectorReader polyReader(Object colType);
 
- Collection<? extends IVectorReader> metadataReaders();
+ IVectorReader metadataReader();
 
  IVectorReader listElementReader();
 

core/src/main/java/xtdb/vector/IndirectVectorReader.java

@@ -168,21 +168,12 @@ public IVectorReader legReader(Keyword legKey) {
  return new IndirectVectorReader(reader.legReader(legKey), indirection);
  }
 
- IndirectVectorReader removeNegativeIndices() {
+ @Override
+ public IVectorReader metadataReader() {
  var idxs = IntStream.range(0, indirection.valueCount()).map(indirection::getIndex).filter(i -> i >= 0).toArray();
  return new IndirectVectorReader(reader, new Selection(idxs));
  }
 
- @Override
- public Collection<? extends IVectorReader> metadataReaders() {
- var legs = reader.legs();
- if (legs == null) return null;
-
- return legs.stream()
- .map(leg -> new IndirectVectorReader(reader.legReader(leg), indirection).removeNegativeIndices())
- .toList();
- }
-
  @Override
  public IVectorReader copyTo(ValueVector vector) {
  IVectorWriter writer = (IVectorWriter) VEC_TO_WRITER.invoke(vector);

core/src/main/java/xtdb/vector/RenamedVectorReader.java

@@ -191,8 +191,8 @@ public IPolyVectorReader polyReader(Object colType) {
  }
 
  @Override
- public Collection<? extends IVectorReader> metadataReaders() {
- return reader.metadataReaders();
+ public IVectorReader metadataReader() {
+ return new RenamedVectorReader(reader.metadataReader(), colName);
  }
 
  @Override

core/src/main/java/xtdb/vector/ValueVectorReader.java

@@ -225,8 +225,8 @@ public IPolyVectorReader polyReader(Object colType) {
  }
 
  @Override
- public Collection<? extends IVectorReader> metadataReaders() {
- return null;
+ public IVectorReader metadataReader() {
+ return this;
  }
 
  @Override
@@ -709,11 +709,6 @@ public IVectorReader legReader(Keyword legKey) {
  public Collection<Keyword> legs() {
  return Arrays.stream(legs).toList();
  }
-
- @Override
- public Collection<? extends IVectorReader> metadataReaders() {
- return Arrays.stream(legReaders).map(IndirectVectorReader::removeNegativeIndices).toList();
- }
  }
 
  public static IVectorReader denseUnionVector(DenseUnionVector v) {

src/test/resources/xtdb/indexer-test/can-build-chunk-as-arrow-ipc-file-format/tables/device_info/chunks/trie-c00.arrow.json

@@ -22,7 +22,7 @@
  "name" : "list"
  },
  "children" : [{
- "name" : "$data",
+ "name" : "$data$",
  "nullable" : true,
  "type" : {
  "name" : "int",
@@ -46,6 +46,110 @@
  "isSigned" : true
  },
  "children" : [ ]
+ },{
+ "name" : "columns",
+ "nullable" : false,
+ "type" : {
+ "name" : "list"
+ },
+ "children" : [{
+ "name" : "$data$",
+ "nullable" : false,
+ "type" : {
+ "name" : "struct"
+ },
+ "children" : [{
+ "name" : "col-name",
+ "nullable" : false,
+ "type" : {
+ "name" : "utf8"
+ },
+ "children" : [ ]
+ },{
+ "name" : "root-col?",
+ "nullable" : false,
+ "type" : {
+ "name" : "bool"
+ },
+ "children" : [ ]
+ },{
+ "name" : "count",
+ "nullable" : false,
+ "type" : {
+ "name" : "int",
+ "bitWidth" : 64,
+ "isSigned" : true
+ },
+ "children" : [ ]
+ },{
+ "name" : "types",
+ "nullable" : false,
+ "type" : {
+ "name" : "struct"
+ },
+ "children" : [{
+ "name" : "timestamp-tz-micro-utc",
+ "nullable" : true,
+ "type" : {
+ "name" : "struct"
+ },
+ "children" : [{
+ "name" : "min",
+ "nullable" : true,
+ "type" : {
+ "name" : "timestamp",
+ "unit" : "MICROSECOND",
+ "timezone" : "UTC"
+ },
+ "children" : [ ]
+ },{
+ "name" : "max",
+ "nullable" : true,
+ "type" : {
+ "name" : "timestamp",
+ "unit" : "MICROSECOND",
+ "timezone" : "UTC"
+ },
+ "children" : [ ]
+ }]
+ },{
+ "name" : "fixed-size-binary",
+ "nullable" : true,
+ "type" : {
+ "name" : "bool"
+ },
+ "children" : [ ]
+ },{
+ "name" : "utf8",
+ "nullable" : true,
+ "type" : {
+ "name" : "struct"
+ },
+ "children" : [{
+ "name" : "min",
+ "nullable" : true,
+ "type" : {
+ "name" : "utf8"
+ },
+ "children" : [ ]
+ },{
+ "name" : "max",
+ "nullable" : true,
+ "type" : {
+ "name" : "utf8"
+ },
+ "children" : [ ]
+ }]
+ }]
+ },{
+ "name" : "bloom",
+ "nullable" : true,
+ "type" : {
+ "name" : "binary"
+ },
+ "children" : [ ]
+ }]
+ }]
  }]
  }]
  }]
@@ -66,7 +170,7 @@
  "VALIDITY" : [ ],
  "OFFSET" : [0],
  "children" : [{
- "name" : "$data",
+ "name" : "$data$",
  "count" : 0,
  "VALIDITY" : [ ],
  "DATA" : [ ]
@@ -80,6 +184,81 @@
  "count" : 1,
  "VALIDITY" : [1],
  "DATA" : [0]
+ },{
+ "name" : "columns",
+ "count" : 1,
+ "VALIDITY" : [1],
+ "OFFSET" : [0,9],
+ "children" : [{
+ "name" : "$data$",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "children" : [{
+ "name" : "col-name",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "OFFSET" : [0,14,20,33,44,49,56,61,72,84],
+ "DATA" : ["xt$system_from","xt$iid","xt$valid_from","xt$valid_to","xt$id","os_name","model","api_version","manufacturer"]
+ },{
+ "name" : "root-col?",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "DATA" : [1,1,1,1,1,1,1,1,1]
+ },{
+ "name" : "count",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "DATA" : ["2","2","2","2","2","2","2","2","2"]
+ },{
+ "name" : "types",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "children" : [{
+ "name" : "timestamp-tz-micro-utc",
+ "count" : 9,
+ "VALIDITY" : [1,0,1,1,0,0,0,0,0],
+ "children" : [{
+ "name" : "min",
+ "count" : 9,
+ "VALIDITY" : [1,0,1,1,0,0,0,0,0],
+ "DATA" : [1577836800000000,0,1577836800000000,253402300799999999,0,0,0,0,0]
+ },{
+ "name" : "max",
+ "count" : 9,
+ "VALIDITY" : [1,0,1,1,0,0,0,0,0],
+ "DATA" : [1577923200000000,0,1577923200000000,253402300799999999,0,0,0,0,0]
+ }]
+ },{
+ "name" : "fixed-size-binary",
+ "count" : 9,
+ "VALIDITY" : [0,1,0,0,0,0,0,0,0],
+ "DATA" : [0,1,0,0,0,0,0,0,0]
+ },{
+ "name" : "utf8",
+ "count" : 9,
+ "VALIDITY" : [0,0,0,0,1,1,1,1,1],
+ "children" : [{
+ "name" : "min",
+ "count" : 9,
+ "VALIDITY" : [0,0,0,0,1,1,1,1,1],
+ "OFFSET" : [0,0,0,0,0,22,27,34,36,42],
+ "DATA" : ["","","","","device-info-demo000000","6.0.1","mustang","23","iobeam"]
+ },{
+ "name" : "max",
+ "count" : 9,
+ "VALIDITY" : [0,0,0,0,1,1,1,1,1],
+ "OFFSET" : [0,0,0,0,0,22,27,32,34,40],
+ "DATA" : ["","","","","device-info-demo000001","6.0.1","pinto","23","iobeam"]
+ }]
+ }]
+ },{
+ "name" : "bloom",
+ "count" : 9,
+ "VALIDITY" : [1,1,1,1,1,1,1,1,1],
+ "OFFSET" : [0,68,136,204,234,278,316,376,406,444],
+ "DATA" : ["3a300000060000000200000004000000090000000b0000000c0000000f000000380000003a0000003c0000003e00000040000000420000000af41c3b4c0193972a6e08db","3a300000060000000000000001000000080000000c0000000e0000000f000000380000003a0000003c0000003e000000400000004200000072af47c4c262fe1e38673d01","3a300000060000000200000004000000090000000b0000000c0000000f000000380000003a0000003c0000003e00000040000000420000000af41c3b4c0193972a6e08db","3a300000020000000600010007000000180000001c0000005563a4bff31b","3a3000000300000001000000080003000c00000020000000220000002a000000e92f09380a38f066d695f9b3","3a30000003000000060000000700000008000000200000002200000024000000ffad90682123","3a3000000500000004000100050000000b0000000d0000000e000000300000003400000036000000380000003a00000068c95ff09f3d1f57ff16b13b","3a300000020000000000010008000000180000001c0000006d07676cea39","3a3000000300000005000000090000000f000000200000002200000024000000f3d627730da5"]
+ }]
+ }]
  }]
  }]
  }]