-
Notifications
You must be signed in to change notification settings - Fork 5.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
util: add PreAlloc4Row and Insert for Chunk and List #7916
Changes from 2 commits
af650d0
577f676
837a517
6eb77ad
3b239b5
f91ebc0
79e6eb0
52497c0
b73c2e9
0ac6476
f5e4088
af8fc82
e59f8d9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,6 +15,7 @@ package chunk | |
|
||
import ( | ||
"encoding/binary" | ||
"reflect" | ||
"unsafe" | ||
|
||
"github.com/cznic/mathutil" | ||
|
@@ -277,6 +278,60 @@ func (c *Chunk) AppendPartialRow(colIdx int, row Row) { | |
} | ||
} | ||
|
||
// PreAlloc4Row pre-allocates the memory space for a Row. | ||
// The null elem info will be pre-written. | ||
func (c *Chunk) PreAlloc4Row(row Row) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we preallocate the memory for a batch of rows? The memory grow stratagem may allocate a lot of unused memory. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. how about:
// PreAlloc pre-allocates the memory space in a Chunk to store the Row.
// NOTE:
// 1. The Chunk must be empty or holds no useful data.
// 2. The schema of the Row must be the same with the Chunk.
// 3. This API is paired with the `Insert()` function, which inserts all the
// rows into the Chunk after the pre-allocation. |
||
for i, rowCol := range row.c.columns { | ||
chkCol := c.columns[i] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. how about:
|
||
chkCol.appendNullBitmap(!rowCol.isNull(row.idx)) | ||
if rowCol.isFixed() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. how about: elemLen := len(rowCol.elemBuf)
if !rowCol.isFixed() {
elemLen = int(rowCol.offsets[row.idx+1] - rowCol.offsets[row.idx])
chkCol.offsets = append(chkCol.offsets, len(chkCol.data)+elemLen)
}
if len(chkCol.data)+elemLen >= cap(chkCol.data) {
... |
||
elemLen := len(rowCol.elemBuf) | ||
if len(chkCol.data)+elemLen >= cap(chkCol.data) { | ||
chkCol.data = make([]byte, len(chkCol.data)+elemLen, 2*cap(chkCol.data)) | ||
} else { | ||
(*reflect.SliceHeader)(unsafe.Pointer(&chkCol.data)).Len = len(chkCol.data) + elemLen | ||
} | ||
} else { | ||
elemLen := int(rowCol.offsets[row.idx+1] - rowCol.offsets[row.idx]) | ||
if len(chkCol.data)+elemLen >= cap(chkCol.data) { | ||
chkCol.data = make([]byte, len(chkCol.data)+elemLen, 2*cap(chkCol.data)) | ||
} else { | ||
(*reflect.SliceHeader)(unsafe.Pointer(&chkCol.data)).Len = len(chkCol.data) + elemLen | ||
} | ||
chkCol.offsets = append(chkCol.offsets, int32(len(chkCol.data))) | ||
} | ||
chkCol.length++ | ||
} | ||
c.numVirtualRows++ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why do we increase |
||
} | ||
|
||
// Insert inserts `row` on the position specified by `rowIdx`. | ||
// Note: Insert will cover the origin data, it should be called after | ||
// PreAlloc4Row. | ||
func (c *Chunk) Insert(rowIdx int, row Row) { | ||
// Check data length between row and the origin data for every column. | ||
// Cover the origin data if the upper check is valid. | ||
for i, rowCol := range row.c.columns { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. s/rowCol/srcCol/ |
||
chkCol := c.columns[i] | ||
if chkCol.isFixed() != rowCol.isFixed() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this check can be removed |
||
panic("unexcepted error happens during Chunk.Insert") | ||
} | ||
var srcStart, srcEnd, destStart, destEnd int | ||
if rowCol.isFixed() { | ||
srcElemLen, destElemLen := len(rowCol.elemBuf), len(chkCol.elemBuf) | ||
srcStart, destStart = row.idx*srcElemLen, rowIdx*destElemLen | ||
srcEnd, destEnd = srcStart+srcElemLen, destStart+destElemLen | ||
alivxxx marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} else { | ||
srcStart, srcEnd = int(rowCol.offsets[row.idx]), int(rowCol.offsets[row.idx+1]) | ||
destStart, destEnd = int(chkCol.offsets[rowIdx]), int(chkCol.offsets[rowIdx+1]) | ||
} | ||
if destEnd-destStart != srcEnd-srcStart { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we really need to add this length check? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In practice, this should be promised by the caller, as well as line 316. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this check can be removed. |
||
panic("unexcepted error happens during Chunk.Insert") | ||
} | ||
copy(chkCol.data[destStart:destEnd], rowCol.data[srcStart:srcEnd]) | ||
} | ||
} | ||
|
||
// Append appends rows in [begin, end) in another Chunk to a Chunk. | ||
func (c *Chunk) Append(other *Chunk, begin, end int) { | ||
for colID, src := range other.columns { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -140,6 +140,32 @@ func (l *List) Reset() { | |
l.consumedIdx = -1 | ||
} | ||
|
||
// PreAlloc4Row pre-allocate the storage memory for a Row. | ||
func (l *List) PreAlloc4Row(row Row) (ptr RowPtr) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the comment of this function should also be updated. |
||
chkIdx := len(l.chunks) - 1 | ||
if chkIdx == -1 || l.chunks[chkIdx].NumRows() >= l.chunks[chkIdx].Capacity() || chkIdx == l.consumedIdx { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this check can be removed: |
||
newChk := l.allocChunk() | ||
l.chunks = append(l.chunks, newChk) | ||
if chkIdx != l.consumedIdx { | ||
l.memTracker.Consume(l.chunks[chkIdx].MemoryUsage()) | ||
l.consumedIdx = chkIdx | ||
} | ||
chkIdx++ | ||
} | ||
chk := l.chunks[chkIdx] | ||
rowIdx := chk.NumRows() | ||
chk.PreAlloc4Row(row) | ||
l.length++ | ||
return RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)} | ||
} | ||
|
||
// Insert inserts `row` on the position specified by `ptr`. | ||
// Note: Insert will cover the origin data, it should be called after | ||
// PreAlloc4Row. | ||
func (l *List) Insert(ptr RowPtr, row Row) { | ||
l.chunks[ptr.ChkIdx].Insert(int(ptr.RowIdx), row) | ||
} | ||
|
||
// ListWalkFunc is used to walk the list. | ||
// If error is returned, it will stop walking. | ||
type ListWalkFunc = func(row Row) error | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
row
andc
be the same?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, but it's should be promised by the caller.