Skip to content

Commit

Permalink
Fix incorrent bytes to calculate length of ziplist/listpack 32bit str…
Browse files Browse the repository at this point in the history 
…ing (#2124)

Currently, it calculates the length of 32-bit string from position 0 to 3,
but the right position should be 1 to 4 in ziplist/listpack.
So it will retrieve a wrong length and throw an error.
  • Loading branch information
@git-hulk
git-hulk authored Feb 28, 2024
1 parent 019faed commit 67208e6
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 4 deletions.
4 changes: 2 additions & 2 deletions src/storage/rdb_listpack.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,8 @@ StatusOr<std::string> ListPack::Next() {
pos_ += value_len + encodeBackLen(value_len + 2);
} else if ((c & ListPack32BitStringMask) == ListPack32BitString) { // 32bit string
GET_OR_RET(peekOK(5));
value_len = (static_cast<uint32_t>(data[pos_])) | (static_cast<uint32_t>(data[pos_ + 1]) << 8) |
(static_cast<uint32_t>(data[pos_ + 2]) << 16) | (static_cast<uint32_t>(data[pos_ + 3]) << 24);
value_len = (static_cast<uint32_t>(data[pos_ + 1])) | (static_cast<uint32_t>(data[pos_ + 2]) << 8) |
(static_cast<uint32_t>(data[pos_ + 3]) << 16) | (static_cast<uint32_t>(data[pos_ + 4]) << 24);
// skip 5byte encoding type
pos_ += 5;
GET_OR_RET(peekOK(value_len));
Expand Down
4 changes: 2 additions & 2 deletions src/storage/rdb_ziplist.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ StatusOr<std::string> ZipList::Next() {
} else if ((encoding) == ZIP_STR_32B) {
GET_OR_RET(peekOK(5));
len_bytes = 5;
len = (static_cast<uint32_t>(data[pos_]) << 24) | (static_cast<uint32_t>(data[pos_ + 1]) << 16) |
(static_cast<uint32_t>(data[pos_ + 2]) << 8) | static_cast<uint32_t>(data[pos_ + 3]);
len = (static_cast<uint32_t>(data[pos_ + 1]) << 24) | (static_cast<uint32_t>(data[pos_ + 2]) << 16) |
(static_cast<uint32_t>(data[pos_ + 3]) << 8) | static_cast<uint32_t>(data[pos_ + 4]);
} else {
return {Status::NotOK, "invalid ziplist encoding"};
}
Expand Down
61 changes: 61 additions & 0 deletions tests/gocase/unit/rdb/rdb_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package rdb

import (
"context"
"encoding/base64"
"io"
"os"
"path/filepath"
"strings"
"testing"

"github.com/apache/kvrocks/tests/gocase/util"
"github.com/stretchr/testify/require"
)

func TestLoadRDB(t *testing.T) {

srv := util.StartServer(t, map[string]string{})
defer srv.Close()

ctx := context.Background()
client := srv.NewClient()
defer func() { require.NoError(t, client.Close()) }()

t.Run("key with ziplist 32bit string", func(t *testing.T) {
encodedData := "UkVESVMwMDA5+glyZWRpcy12ZXIGNi4yLjEw+gpyZWRpcy1iaXRzwED6BWN0aW1lwtbG3GX6CHVzZWQtbWVtwsA9FAD6DGFvZi1wcmVhbWJsZcAA/gD7AQAOBEFCQ0QNwxdB6wTrAQAA6CADBPAAAAAC4P8B4MsBAQD/w0HNgAAAi5EOkYsAAAoAAAABAACAAACLIAMBwH/g/wPhDQfg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgYwBP++D/AOD/AOD/AOD/AEAA5P8n4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4LcAAcB/4LfB4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4JUA7f+/4P8A4F8A4v934P8A4P8A4P8A4KsA5PPTQPtAA+D/AOD/AOD/AOC7AOP/3+D/AOD/AOD/AOAnAOT/T+D/AOD/AOD/AOD/AOD/AOD/AODTAOj/E+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOCbAPn/Y+D/AOD/AOD/AOD/AODnAOb/F+BvAOH/f+D/AOD/AOD/AOD/AOD/AOD/AOD/AOC3AOj//+D/AOAcAAEA/8NBt4AAAIKRDpGCAAAKAAAAAQAAgAAAgiAD4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4AcAAcB/4AcR4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4BUATp9AA+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AMAA7f9z4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4OsA86+DQLfgMwPg/wDg/wDg/wDgKwDjF4fg/wDg/wDg/wDg/wDg/wDgNwDlR4dAT+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOCnAPv/g+D/AOD/AOD/AOBnAOT/j+D/AOD/AOD/AOD/AOD/AOD/AOBgAAEA/8NB3IAAAIuRDpGLAAAKAAAAAQAAgAAAiyAD4P8A4FMAAcB/4FNd4P8A4P8A4P8A4P8A5P9/4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4MkA9P/H4P8A4P8A4GsA4/+L4P8A4P8A4P8A4P8A4P8A4CsA5v9j4AsA4f8b4P8A4FcA4v9v4P8A4P8A4P8A4P8A4P8A4FcA5v+P4P8A4P8A4P8A4P8A4P8A4P8A4P8A4DMA6Ht74HuD4PMA4f9/4D8A4f9P4A8A4Ycf4IeP4AMAQJtAA0AAwAfg/wDg/wDg/wDgHwDj/0fg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgKwD0/9Pg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg5wD7/7/g/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg+wD1/6Pg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgTAABAP/DQa+AAACHEQ0RhwAACgAAAAEAAIAAACAM4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4DwAAcB/4DxG4P8A4OgA4v8/4P8A4P8A4P8A4P8A4P8A4IcA5v+/4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4GcA7//n4P8A4P8A4P8A4P8A4McAxfdAB0AD4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4KAAAQD/w0GzgAAAi5EOkYsAAAoAAAABAACAAACLIAPg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg2wABwH/g2+Xg/wDg/wDg/wDg/wDg/wDg/wDg/wDgNQDo/1vg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgRAABAP/DQcSAAACHEQ0RhwAACgAAAAEAAIAAACAM4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4CwAAcB/4D0DwABAT+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOAzAO//t+D/AOD/AOD/AOD/AOD/AOATAEZLQAPg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgEwD+/wfg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgJwDt/5fgtwDBx0AHQANAAMAH4P8A4P8A4P8A4P8A4P8A4P8A4JsA5v/b4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4EAAAQD/w0GzgAAAi5EOkYsAAAoAAAABAACAAACLIAPg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgOwABwH/gAQPg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg9AABAP/DQfmAAACLkQ6RiwAACgAAAAEAAIAAAIsgA+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOCPAAHAf+CPmeD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOAFAO3/B+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AEAAT3tAA0AAQAdAA+AbAOAfJ+BXAECHQAPgFwBAI0AD4AMAQA9AA+AvAEA7QAPgFwBAI+AjA+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOBXAOkj0+AXK+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOC3ANaH4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4M8ATDfg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgCwDz/6/g/wDg/wDg/wDg/wDg/wDg/wDgjwDn/8/g/wDg/wDg/wDg/wDg/wDgKwDm/2Pg/wDg/wDg/wDgtwDk/9/g/wDg/wDg/wDg/wDg/wDgrAABAP/DQbaAAACHEQ0RhwAACgAAAAEAAIAAACAM4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4DwAAcB/4DxG4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4FwAzQtAB+AnA+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AODHAAHAf+DH0eD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AODpAAHAf+Dp8+D/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOBQAAEA/8NBzoAAAIuRDpGLAAAKAAAAAQAAgAAAiyAD4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4OsAAcB/4Mn1QNNAA+D/AOD/AOBLAEJnQAPg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgJwDu/6Pg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDg/wDgJwBTx+APA0AAQBtAAMAH4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4P8A4PcAAcB/4fcB4KYAAQD/w0GrgAAAhxENEYcAAAoAAAABAACAAAAgDOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AOD/AODoAAHAf+Do8uD/AOD/AOD/AOD/AOD/AOD/AOD/AOBEAEh34P8D4UsH4P8A4P8A4ktj4P9T4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4f8H4RgHAX//wxpCxwTHAgAAxCADBF4BAAAC4P8B4P8B4J8BAQD//zD9AvrHFdOW"
decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedData))
decodedData, err := io.ReadAll(decoder)
require.NoError(t, err)

rdbFileName := "dump-with-32bit-string.rdb"
require.NoError(t, os.WriteFile(rdbFileName, decodedData, 0644))
defer func() {
_ = os.Remove(rdbFileName)
}()

absFilePath, err := filepath.Abs(rdbFileName)
require.NoError(t, err)
require.NoError(t, client.Do(ctx, "RDB", "LOAD", absFilePath).Err())
require.EqualValues(t, 601, client.LLen(ctx, "ABCD").Val())
})
}
Loading

0 comments on commit 67208e6

Please sign in to comment.