forked from matrixorigin/matrixone
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
(tae): update mock index file and some index codes (#15)
Signed-off-by: asuka <312856403@qq.com>
- Loading branch information
Showing
9 changed files
with
280 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
...engine/tae/index/access/accessif/iface.go → ...vm/engine/tae/index/access/acif/ifaces.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| package accessif | ||
| package acif | ||
|
|
||
| import "github.com/matrixorigin/matrixone/pkg/container/vector" | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,171 @@ | ||
| package io | ||
|
|
||
| import ( | ||
| "github.com/RoaringBitmap/roaring" | ||
| "github.com/matrixorigin/matrixone/pkg/container/types" | ||
| "github.com/matrixorigin/matrixone/pkg/container/vector" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer/base" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/dataio" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index/basic" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index/common" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index/common/errors" | ||
| ) | ||
|
|
||
| type blockZoneMapIndexNode struct { | ||
| *buffer.Node | ||
| mgr base.INodeManager | ||
| host dataio.BlockFile | ||
| host dataio.IndexFile | ||
| meta *common.IndexMeta | ||
| inner *basic.ZoneMap | ||
| } | ||
|
|
||
| func newBlockZoneMapIndexNode(mgr base.INodeManager, host dataio.BlockFile, meta *common.IndexMeta) *blockZoneMapIndexNode { | ||
| func newBlockZoneMapIndexNode(mgr base.INodeManager, host dataio.IndexFile, meta *common.IndexMeta) *blockZoneMapIndexNode { | ||
| impl := new(blockZoneMapIndexNode) | ||
| impl.Node = buffer.NewNode(impl, mgr, host.AllocIndexNodeId(), uint64(meta.Size)) | ||
| impl.LoadFunc = impl.OnLoad | ||
| impl.UnloadFunc = impl.OnUnload | ||
| impl.DestroyFunc = impl.OnDestroy | ||
| impl.host = host | ||
| impl.meta = meta | ||
| impl.mgr = mgr | ||
| mgr.RegisterNode(impl) | ||
| return impl | ||
| } | ||
|
|
||
| func (n *blockZoneMapIndexNode) OnLoad() { | ||
| if n.inner != nil { | ||
| // no-op | ||
| return | ||
| } | ||
| var err error | ||
| startOffset := n.meta.StartOffset | ||
| size := n.meta.Size | ||
| compressTyp := n.meta.CompType | ||
| data := n.host.Read(startOffset, size) | ||
| rawSize := n.meta.RawSize | ||
| buf := make([]byte, rawSize) | ||
| if err = common.Decompress(data, buf, compressTyp); err != nil { | ||
| panic(err) | ||
| } | ||
| n.inner, err = basic.NewZoneMapFromSource(buf) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func (n *blockZoneMapIndexNode) OnUnload() { | ||
| if n.inner == nil { | ||
| // no-op | ||
| return | ||
| } | ||
| n.inner = nil | ||
| } | ||
|
|
||
| func (n *blockZoneMapIndexNode) OnDestroy() { | ||
| // no-op | ||
| } | ||
|
|
||
| func (n *blockZoneMapIndexNode) Close() error { | ||
| // no-op | ||
| return nil | ||
| } | ||
|
|
||
| type BlockZoneMapIndexReader struct { | ||
| inode *blockZoneMapIndexNode | ||
| } | ||
|
|
||
| func NewBlockZoneMapIndexReader() *BlockZoneMapIndexReader { | ||
| return &BlockZoneMapIndexReader{} | ||
| } | ||
|
|
||
| func (reader *BlockZoneMapIndexReader) Init(mgr base.INodeManager, host dataio.IndexFile, meta *common.IndexMeta) error { | ||
| reader.inode = newBlockZoneMapIndexNode(mgr, host, meta) | ||
| return nil | ||
| } | ||
|
|
||
| func (reader *BlockZoneMapIndexReader) MayContainsAnyKeys(keys *vector.Vector) (bool, *roaring.Bitmap, error) { | ||
| handle := reader.inode.mgr.Pin(reader.inode) | ||
| defer handle.Close() | ||
| return handle.GetNode().(*blockZoneMapIndexNode).inner.MayContainsAnyKeys(keys) | ||
| } | ||
|
|
||
| func (reader *BlockZoneMapIndexReader) MayContainsKey(key interface{}) (bool, error) { | ||
| handle := reader.inode.mgr.Pin(reader.inode) | ||
| defer handle.Close() | ||
| return handle.GetNode().(*blockZoneMapIndexNode).inner.MayContainsKey(key) | ||
| } | ||
|
|
||
| type BlockZoneMapIndexWriter struct { | ||
| cType common.CompressType | ||
| host dataio.IndexFile | ||
| inner *basic.ZoneMap | ||
| colIdx uint16 | ||
| } | ||
|
|
||
| func NewBlockZoneMapIndexWriter() *BlockZoneMapIndexWriter { | ||
| return &BlockZoneMapIndexWriter{} | ||
| } | ||
|
|
||
| func (writer *BlockZoneMapIndexWriter) Init(host dataio.IndexFile, cType common.CompressType, colIdx uint16) error { | ||
| writer.host = host | ||
| writer.cType = cType | ||
| writer.colIdx = colIdx | ||
| return nil | ||
| } | ||
|
|
||
| // TODO: add more types of index node, and integrate with buffer manager | ||
| func (writer *BlockZoneMapIndexWriter) Finalize() (*common.IndexMeta, error) { | ||
| if writer.inner == nil { | ||
| panic("unexpected error") | ||
| } | ||
| appender := writer.host | ||
| meta := common.NewEmptyIndexMeta() | ||
| meta.SetIndexType(common.BlockZoneMapIndex) | ||
| meta.SetCompressType(writer.cType) | ||
| meta.SetIndexedColumn(writer.colIdx) | ||
|
|
||
| var startOffset uint32 | ||
| iBuf, err := writer.inner.Marshal() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| rawSize := uint32(len(iBuf)) | ||
| cBuf := common.Compress(iBuf, writer.cType) | ||
| exactSize := uint32(len(cBuf)) | ||
| meta.SetSize(rawSize, exactSize) | ||
| startOffset, err = appender.Append(cBuf) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| meta.SetStartOffset(startOffset) | ||
| return meta, nil | ||
| } | ||
|
|
||
| func (writer *BlockZoneMapIndexWriter) AddValues(values *vector.Vector) error { | ||
| typ := values.Typ | ||
| if writer.inner == nil { | ||
| writer.inner = basic.NewZoneMap(typ, nil) | ||
| } else { | ||
| if writer.inner.GetType() != typ { | ||
| return errors.ErrTypeMismatch | ||
| } | ||
| } | ||
| if err := writer.inner.BatchUpdate(values, 0, -1); err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (writer *BlockZoneMapIndexWriter) SetMinMax(min, max interface{}, typ types.Type) error { | ||
| if writer.inner == nil { | ||
| writer.inner = basic.NewZoneMap(typ, nil) | ||
| } else { | ||
| if writer.inner.GetType() != typ { | ||
| return errors.ErrTypeMismatch | ||
| } | ||
| } | ||
| writer.inner.SetMin(min) | ||
| writer.inner.SetMax(max) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package io | ||
|
|
||
| import ( | ||
| "github.com/RoaringBitmap/roaring" | ||
| "github.com/matrixorigin/matrixone/pkg/container/types" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/dataio" | ||
| "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index/common" | ||
| "github.com/stretchr/testify/require" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestBlockZoneMapIndex(t *testing.T) { | ||
| bufManager := buffer.NewNodeManager(1024 * 1024, nil) | ||
| file := dataio.MockIndexFile() | ||
| cType := common.Plain | ||
| typ := types.Type{Oid: types.T_int32} | ||
| pkColIdx := uint16(0) | ||
| var err error | ||
| var meta *common.IndexMeta | ||
| var res bool | ||
| var ans *roaring.Bitmap | ||
|
|
||
| writer := NewBlockZoneMapIndexWriter() | ||
| err = writer.Init(file, cType, pkColIdx) | ||
| require.NoError(t, err) | ||
|
|
||
| keys := common.MockVec(typ, 1000, 0) | ||
| err = writer.AddValues(keys) | ||
| require.NoError(t, err) | ||
|
|
||
| meta, err = writer.Finalize() | ||
| require.NoError(t, err) | ||
|
|
||
| reader := NewBlockZoneMapIndexReader() | ||
| err = reader.Init(bufManager, file, meta) | ||
| require.NoError(t, err) | ||
|
|
||
| //t.Log(bufManager.String()) | ||
|
|
||
| res, err = reader.MayContainsKey(int32(500)) | ||
| require.NoError(t, err) | ||
| require.True(t, res) | ||
|
|
||
| res, err = reader.MayContainsKey(int32(1000)) | ||
| require.NoError(t, err) | ||
| require.False(t, res) | ||
|
|
||
| keys = common.MockVec(typ, 100, 1000) | ||
| res, ans, err = reader.MayContainsAnyKeys(keys) | ||
| require.NoError(t, err) | ||
| require.False(t, res) | ||
| require.Equal(t, uint64(0), ans.GetCardinality()) | ||
|
|
||
| keys = common.MockVec(typ, 100, 0) | ||
| res, ans, err = reader.MayContainsAnyKeys(keys) | ||
| require.NoError(t, err) | ||
| require.True(t, res) | ||
| require.Equal(t, uint64(100), ans.GetCardinality()) | ||
|
|
||
| //t.Log(bufManager.String()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.