From 3b84c51699334c823e07c11173a7497e2953e43d Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Tue, 21 Mar 2023 18:09:41 +0800 Subject: [PATCH 01/26] Implement the new encoding with 64bit size and expire time in milliseconds --- src/cluster/slot_migrate.cc | 8 +- src/commands/cmd_key.cc | 8 +- src/storage/batch_extractor.cc | 4 +- src/storage/redis_db.cc | 22 +++--- src/storage/redis_db.h | 4 +- src/storage/redis_metadata.cc | 109 +++++++++++++++++++-------- src/storage/redis_metadata.h | 16 ++-- src/types/redis_bitmap.cc | 14 ++-- src/types/redis_string.cc | 10 +-- tests/cppunit/types/metadata_test.cc | 6 +- tests/cppunit/types/string_test.cc | 10 +-- utils/kvrocks2redis/parser.cc | 6 +- utils/kvrocks2redis/parser.h | 2 +- 13 files changed, 138 insertions(+), 81 deletions(-) diff --git a/src/cluster/slot_migrate.cc b/src/cluster/slot_migrate.cc index b62562aceaf..467df32777e 100644 --- a/src/cluster/slot_migrate.cc +++ b/src/cluster/slot_migrate.cc @@ -619,7 +619,7 @@ Status SlotMigrate::MigrateSimpleKey(const rocksdb::Slice &key, const Metadata & bytes.substr(Redis::STRING_HDR_SIZE, bytes.size() - Redis::STRING_HDR_SIZE)}; if (metadata.expire > 0) { command.emplace_back("EXAT"); - command.emplace_back(std::to_string(metadata.expire)); + command.emplace_back(std::to_string(metadata.expire / 1000)); } *restore_cmds += Redis::MultiBulkString(command, false); current_pipeline_size_++; @@ -727,7 +727,8 @@ Status SlotMigrate::MigrateComplexKey(const rocksdb::Slice &key, const Metadata // Add TTL for complex key if (metadata.expire > 0) { - *restore_cmds += Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire)}, false); + *restore_cmds += + Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire / 1000)}, false); current_pipeline_size_++; } @@ -803,7 +804,8 @@ Status SlotMigrate::MigrateStream(const Slice &key, const StreamMetadata &metada // Add TTL if (metadata.expire > 0) { - *restore_cmds += Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire)}, false); + *restore_cmds += + Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire / 1000)}, false); current_pipeline_size_++; } diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 0df304c0998..30e52292151 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -18,6 +18,8 @@ * */ +#include + #include "commander.h" #include "commands/ttl_util.h" #include "error_constants.h" @@ -69,7 +71,7 @@ class CommandTTL : public Commander { public: Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - int ttl = 0; + int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (s.ok()) { *output = Redis::Integer(ttl); @@ -84,7 +86,7 @@ class CommandPTTL : public Commander { public: Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - int ttl = 0; + int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (!s.ok()) return {Status::RedisExecErr, s.ToString()}; @@ -235,7 +237,7 @@ class CommandPExpireAt : public Commander { class CommandPersist : public Commander { public: Status Execute(Server *svr, Connection *conn, std::string *output) override { - int ttl = 0; + int64_t ttl = 0; Redis::Database redis(svr->storage_, conn->GetNamespace()); auto s = redis.TTL(args_[1], &ttl); if (!s.ok()) return {Status::RedisExecErr, s.ToString()}; diff --git a/src/storage/batch_extractor.cc b/src/storage/batch_extractor.cc index ac57ffdc5a3..4834a7f54b4 100644 --- a/src/storage/batch_extractor.cc +++ b/src/storage/batch_extractor.cc @@ -62,7 +62,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic command_args = {"SET", user_key, value.ToString().substr(5, value.size() - 5)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); if (metadata.expire > 0) { - command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire)}; + command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire / 1000)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); } } else if (metadata.expire > 0) { @@ -74,7 +74,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic } auto cmd = static_cast(*parse_result); if (cmd == kRedisCmdExpire) { - command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire)}; + command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire / 1000)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); } } diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 82f4174b52f..4884bb99672 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -74,7 +74,7 @@ rocksdb::Status Database::GetRawMetadataByUserKey(const Slice &user_key, std::st return GetRawMetadata(ns_key, bytes); } -rocksdb::Status Database::Expire(const Slice &user_key, int timestamp) { +rocksdb::Status Database::Expire(const Slice &user_key, uint64_t timestamp) { std::string ns_key; AppendNamespacePrefix(user_key, &ns_key); @@ -90,7 +90,7 @@ rocksdb::Status Database::Expire(const Slice &user_key, int timestamp) { if (metadata.Type() != kRedisString && metadata.size == 0) { return rocksdb::Status::NotFound("no elements"); } - if (metadata.expire == timestamp) return rocksdb::Status::OK(); + if (metadata.expire / 1000 == timestamp) return rocksdb::Status::OK(); auto buf = std::make_unique(value.size()); memcpy(buf.get(), value.data(), value.size()); @@ -140,7 +140,7 @@ rocksdb::Status Database::Exists(const std::vector &keys, int *ret) { return rocksdb::Status::OK(); } -rocksdb::Status Database::TTL(const Slice &user_key, int *ttl) { +rocksdb::Status Database::TTL(const Slice &user_key, int64_t *ttl) { std::string ns_key; AppendNamespacePrefix(user_key, &ns_key); @@ -154,7 +154,11 @@ rocksdb::Status Database::TTL(const Slice &user_key, int *ttl) { Metadata metadata(kRedisNone, false); metadata.Decode(value); - *ttl = metadata.TTL(); + *ttl = metadata.TTL() / 1000; + if (*ttl < 0) { + *ttl = -2; + } + return rocksdb::Status::OK(); } @@ -196,12 +200,10 @@ void Database::Keys(const std::string &prefix, std::vector *keys, K continue; } if (stats) { - int32_t ttl = metadata.TTL(); + int64_t ttl = metadata.TTL(); stats->n_key++; - if (ttl != -1) { - stats->n_expires++; - if (ttl > 0) ttl_sum += ttl; - } + stats->n_expires++; + if (ttl > 0) ttl_sum += ttl / 1000; } if (keys) { ExtractNamespaceKey(iter->key(), &ns, &user_key, storage_->IsSlotIdEncoded()); @@ -399,7 +401,7 @@ rocksdb::Status Database::Dump(const Slice &user_key, std::vector * infos->emplace_back("version"); infos->emplace_back(std::to_string(metadata.version)); infos->emplace_back("expire"); - infos->emplace_back(std::to_string(metadata.expire)); + infos->emplace_back(std::to_string(metadata.expire / 1000)); infos->emplace_back("size"); infos->emplace_back(std::to_string(metadata.size)); diff --git a/src/storage/redis_db.h b/src/storage/redis_db.h index f19322b7f13..18012bf5aee 100644 --- a/src/storage/redis_db.h +++ b/src/storage/redis_db.h @@ -35,10 +35,10 @@ class Database { rocksdb::Status GetMetadata(RedisType type, const Slice &ns_key, Metadata *metadata); rocksdb::Status GetRawMetadata(const Slice &ns_key, std::string *bytes); rocksdb::Status GetRawMetadataByUserKey(const Slice &user_key, std::string *bytes); - rocksdb::Status Expire(const Slice &user_key, int timestamp); + rocksdb::Status Expire(const Slice &user_key, uint64_t timestamp); rocksdb::Status Del(const Slice &user_key); rocksdb::Status Exists(const std::vector &keys, int *ret); - rocksdb::Status TTL(const Slice &user_key, int *ttl); + rocksdb::Status TTL(const Slice &user_key, int64_t *ttl); rocksdb::Status Type(const Slice &user_key, RedisType *type); rocksdb::Status Dump(const Slice &user_key, std::vector *infos); rocksdb::Status FlushDB(); diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index e4ecd1ad09b..077d6a2661c 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -24,10 +24,12 @@ #include #include +#include #include #include #include "cluster/redis_slot.h" +#include "encoding.h" #include "time_util.h" // 52 bit for microseconds and 11 bit for counter @@ -146,32 +148,37 @@ void ComposeSlotKeyPrefix(const Slice &ns, int slotid, std::string *output) { } Metadata::Metadata(RedisType type, bool generate_version) - : flags((uint8_t)0x0f & type), expire(0), version(0), size(0) { + : flags(uint8_t(0x80) | (uint8_t(0x0f) & type)), expire(0), version(0), size(0) { if (generate_version) version = generateVersion(); } rocksdb::Status Metadata::Decode(const std::string &bytes) { - // flags(1byte) + expire (4byte) - if (bytes.size() < 5) { + Slice input(bytes); + if (!GetFixed8(&input, &flags)) { return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); } - Slice input(bytes); - GetFixed8(&input, &flags); - GetFixed32(&input, reinterpret_cast(&expire)); + + if (!GetExpire(&input)) { + return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + } + if (Type() != kRedisString) { - if (input.size() < 12) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + if (input.size() < 8 + CommonEncodedSize()) { + return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + } GetFixed64(&input, &version); - GetFixed32(&input, &size); + GetFixedCommon(&input, &size); } + return rocksdb::Status::OK(); } void Metadata::Encode(std::string *dst) { PutFixed8(dst, flags); - PutFixed32(dst, (uint32_t)expire); + PutExpire(dst); if (Type() != kRedisString) { PutFixed64(dst, version); - PutFixed32(dst, size); + PutFixedCommon(dst, size); } } @@ -198,17 +205,59 @@ bool Metadata::operator==(const Metadata &that) const { return true; } -RedisType Metadata::Type() const { return static_cast(flags & (uint8_t)0x0f); } +RedisType Metadata::Type() const { return static_cast(flags & uint8_t(0x0f)); } + +bool Metadata::Is64BitEncoded() const { return flags & 0x80; } -int32_t Metadata::TTL() const { - if (expire <= 0) { - return -1; +size_t Metadata::CommonEncodedSize() const { return Is64BitEncoded() ? 8 : 4; } + +bool Metadata::GetFixedCommon(rocksdb::Slice *input, uint64_t *value) const { + if (Is64BitEncoded()) { + return GetFixed64(input, value); + } else { + uint32_t v = 0; + bool res = GetFixed32(input, &v); + *value = v; + return res; } - auto now = Util::GetTimeStamp(); - if (expire < now) { - return -2; +} + +bool Metadata::GetExpire(rocksdb::Slice *input) { + uint64_t v = 0; + + if (!GetFixedCommon(input, &v)) { + return false; } - return int32_t(expire - now); + + if (Is64BitEncoded()) { + expire = v; + } else { + expire = v * 1000; + } + + return true; +} + +void Metadata::PutFixedCommon(std::string *dst, uint64_t value) const { + if (Is64BitEncoded()) { + PutFixed64(dst, value); + } else { + PutFixed32(dst, value); + } +} + +void Metadata::PutExpire(std::string *dst) { + if (Is64BitEncoded()) { + PutFixed64(dst, expire); + } else { + PutFixed32(dst, expire / 1000); + } +} + +int64_t Metadata::TTL() const { + auto now = Util::GetTimeStampMS(); + + return int64_t(expire - now); } timeval Metadata::Time() const { @@ -219,13 +268,10 @@ timeval Metadata::Time() const { return created_at; } -bool Metadata::ExpireAt(int64_t expired_ts) const { +bool Metadata::ExpireAt(uint64_t expired_ts) const { if (Type() != kRedisString && Type() != kRedisStream && size == 0) { return true; } - if (expire <= 0) { - return false; - } return expire < expired_ts; } @@ -245,9 +291,9 @@ rocksdb::Status ListMetadata::Decode(const std::string &bytes) { GetFixed8(&input, &flags); GetFixed32(&input, reinterpret_cast(&expire)); if (Type() != kRedisString) { - if (input.size() < 12) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + if (input.size() < 8 + CommonEncodedSize()) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); GetFixed64(&input, &version); - GetFixed32(&input, &size); + GetFixedCommon(&input, &size); } if (Type() == kRedisList) { if (input.size() < 16) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); @@ -279,21 +325,20 @@ void StreamMetadata::Encode(std::string *dst) { } rocksdb::Status StreamMetadata::Decode(const std::string &bytes) { - // flags(1byte) + expire (4byte) - if (bytes.size() < 5) { + Slice input(bytes); + if (!GetFixed8(&input, &flags)) { + return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + } + if (!GetExpire(&input)) { return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); } - Slice input(bytes); - GetFixed8(&input, &flags); - GetFixed32(&input, reinterpret_cast(&expire)); - - if (input.size() < 12) { + if (input.size() < 8 + CommonEncodedSize()) { return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); } GetFixed64(&input, &version); - GetFixed32(&input, &size); + GetFixedCommon(&input, &size); if (input.size() < 88) { return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index 5d052d07698..c1f38efcf32 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -100,19 +100,25 @@ class InternalKey { class Metadata { public: uint8_t flags; - int expire; + uint64_t expire; uint64_t version; - uint32_t size; + uint64_t size; - public: explicit Metadata(RedisType type, bool generate_version = true); static void InitVersionCounter(); + bool Is64BitEncoded() const; + bool GetFixedCommon(rocksdb::Slice *input, uint64_t *value) const; + bool GetExpire(rocksdb::Slice *input); + void PutFixedCommon(std::string *dst, uint64_t value) const; + void PutExpire(std::string *dst); + RedisType Type() const; - virtual int32_t TTL() const; + size_t CommonEncodedSize() const; + virtual int64_t TTL() const; virtual timeval Time() const; virtual bool Expired() const; - virtual bool ExpireAt(int64_t expired_ts) const; + virtual bool ExpireAt(uint64_t expired_ts) const; virtual void Encode(std::string *dst); virtual rocksdb::Status Decode(const std::string &bytes); bool operator==(const Metadata &that) const; diff --git a/src/types/redis_bitmap.cc b/src/types/redis_bitmap.cc index 2f01164ee0d..bf5fbf115c0 100644 --- a/src/types/redis_bitmap.cc +++ b/src/types/redis_bitmap.cc @@ -193,8 +193,8 @@ rocksdb::Status Bitmap::SetBit(const Slice &user_key, uint32_t offset, bool new_ if (!s.ok() && !s.IsNotFound()) return s; } uint32_t byte_index = (offset / 8) % kBitmapSegmentBytes; - uint32_t used_size = index + byte_index + 1; - uint32_t bitmap_size = std::max(used_size, metadata.size); + uint64_t used_size = index + byte_index + 1; + uint64_t bitmap_size = std::max(used_size, metadata.size); if (byte_index >= value.size()) { // expand the bitmap size_t expand_size = 0; if (byte_index >= value.size() * 2) { @@ -240,9 +240,9 @@ rocksdb::Status Bitmap::BitCount(const Slice &user_key, int64_t start, int64_t s return bitmap_string_db.BitCount(raw_value, start, stop, cnt); } - if (start < 0) start += metadata.size + 1; - if (stop < 0) stop += metadata.size + 1; - if (stop > static_cast(metadata.size)) stop = metadata.size; + if (start < 0) start += static_cast(metadata.size) + 1; + if (stop < 0) stop += static_cast(metadata.size) + 1; + if (stop > static_cast(metadata.size)) stop = static_cast(metadata.size); if (start < 0 || stop <= 0 || start >= stop) return rocksdb::Status::OK(); auto u_start = static_cast(start); @@ -289,8 +289,8 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i return bitmap_string_db.BitPos(raw_value, bit, start, stop, stop_given, pos); } - if (start < 0) start += metadata.size + 1; - if (stop < 0) stop += metadata.size + 1; + if (start < 0) start += static_cast(metadata.size) + 1; + if (stop < 0) stop += static_cast(metadata.size) + 1; if (start < 0 || stop < 0 || start > stop) { *pos = -1; return rocksdb::Status::OK(); diff --git a/src/types/redis_string.cc b/src/types/redis_string.cc index 3024dd0c4e3..79ed41e4ea5 100644 --- a/src/types/redis_string.cc +++ b/src/types/redis_string.cc @@ -162,7 +162,7 @@ rocksdb::Status String::GetEx(const std::string &user_key, std::string *value, i std::string raw_data; Metadata metadata(kRedisString, false); - metadata.expire = expire; + metadata.expire = expire * 1000; metadata.Encode(&raw_data); raw_data.append(value->data(), value->size()); auto batch = storage_->GetWriteBatchBase(); @@ -234,7 +234,7 @@ rocksdb::Status String::SetXX(const std::string &user_key, const std::string &va *ret = 1; std::string raw_value; Metadata metadata(kRedisString, false); - metadata.expire = expire; + metadata.expire = expire * 1000; metadata.Encode(&raw_value); raw_value.append(value); return updateRawValue(ns_key, raw_value); @@ -362,7 +362,7 @@ rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { for (const auto &pair : pairs) { std::string bytes; Metadata metadata(kRedisString, false); - metadata.expire = expire; + metadata.expire = expire * 1000; metadata.Encode(&bytes); bytes.append(pair.value.data(), pair.value.size()); auto batch = storage_->GetWriteBatchBase(); @@ -405,7 +405,7 @@ rocksdb::Status String::MSetNX(const std::vector &pairs, int ttl, in } std::string bytes; Metadata metadata(kRedisString, false); - metadata.expire = expire; + metadata.expire = expire * 1000; metadata.Encode(&bytes); bytes.append(pair.value.data(), pair.value.size()); auto batch = storage_->GetWriteBatchBase(); @@ -451,7 +451,7 @@ rocksdb::Status String::CAS(const std::string &user_key, const std::string &old_ int64_t now = Util::GetTimeStamp(); expire = int(now) + ttl; } - metadata.expire = expire; + metadata.expire = expire * 1000; metadata.Encode(&raw_value); raw_value.append(new_value); auto write_status = updateRawValue(ns_key, raw_value); diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index fa1a7e4dc7a..b82f860db30 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -47,14 +47,14 @@ TEST(InternalKey, EncodeAndDecode) { TEST(Metadata, EncodeAndDeocde) { std::string string_bytes; Metadata string_md(kRedisString); - string_md.expire = 123; + string_md.expire = 123000; string_md.Encode(&string_bytes); Metadata string_md1(kRedisNone); string_md1.Decode(string_bytes); ASSERT_EQ(string_md, string_md1); ListMetadata list_md; list_md.flags = 13; - list_md.expire = 123; + list_md.expire = 123000; list_md.version = 2; list_md.size = 1234; list_md.head = 123; @@ -110,7 +110,7 @@ TEST_F(RedisTypeTest, Expire) { int64_t now = 0; rocksdb::Env::Default()->GetCurrentTime(&now); redis->Expire(key_, int(now + 2)); - int ttl = 0; + int64_t ttl = 0; redis->TTL(key_, &ttl); ASSERT_TRUE(ttl >= 1 && ttl <= 2); sleep(2); diff --git a/tests/cppunit/types/string_test.cc b/tests/cppunit/types/string_test.cc index d9d220ea37e..52f2de7e765 100644 --- a/tests/cppunit/types/string_test.cc +++ b/tests/cppunit/types/string_test.cc @@ -137,7 +137,7 @@ TEST_F(RedisStringTest, GetEmptyValue) { } TEST_F(RedisStringTest, GetSet) { - int ttl = 0; + int64_t ttl = 0; int64_t now = 0; rocksdb::Env::Default()->GetCurrentTime(&now); std::vector values = {"a", "b", "c", "d"}; @@ -177,7 +177,7 @@ TEST_F(RedisStringTest, MSetXX) { string->Set(key_, "test-value"); string->SetXX(key_, "test-value", 3, &ret); EXPECT_EQ(ret, 1); - int ttl = 0; + int64_t ttl = 0; string->TTL(key_, &ttl); EXPECT_TRUE(ttl >= 2 && ttl <= 3); string->Del(key_); @@ -211,7 +211,7 @@ TEST_F(RedisStringTest, MSetNX) { TEST_F(RedisStringTest, MSetNXWithTTL) { int ret = 0; string->SetNX(key_, "test-value", 3, &ret); - int ttl = 0; + int64_t ttl = 0; string->TTL(key_, &ttl); EXPECT_TRUE(ttl >= 2 && ttl <= 3); string->Del(key_); @@ -219,7 +219,7 @@ TEST_F(RedisStringTest, MSetNXWithTTL) { TEST_F(RedisStringTest, SetEX) { string->SetEX(key_, "test-value", 3); - int ttl = 0; + int64_t ttl = 0; string->TTL(key_, &ttl); EXPECT_TRUE(ttl >= 2 && ttl <= 3); string->Del(key_); @@ -274,7 +274,7 @@ TEST_F(RedisStringTest, CAS) { ASSERT_TRUE(status.ok()); EXPECT_EQ(new_value, current_value); - int ttl = 0; + int64_t ttl = 0; string->TTL(key, &ttl); EXPECT_TRUE(ttl >= 9 && ttl <= 10); diff --git a/utils/kvrocks2redis/parser.cc b/utils/kvrocks2redis/parser.cc index 89c6ca2edd7..e2a32548d03 100644 --- a/utils/kvrocks2redis/parser.cc +++ b/utils/kvrocks2redis/parser.cc @@ -59,7 +59,7 @@ Status Parser::ParseFullDB() { return Status::OK(); } -Status Parser::parseSimpleKV(const Slice &ns_key, const Slice &value, int expire) { +Status Parser::parseSimpleKV(const Slice &ns_key, const Slice &value, uint64_t expire) { std::string ns, user_key; ExtractNamespaceKey(ns_key, &ns, &user_key, slot_id_encoded_); @@ -69,7 +69,7 @@ Status Parser::parseSimpleKV(const Slice &ns_key, const Slice &value, int expire if (!s.IsOK()) return s; if (expire > 0) { - command = Redis::Command2RESP({"EXPIREAT", user_key, std::to_string(expire)}); + command = Redis::Command2RESP({"EXPIREAT", user_key, std::to_string(expire / 1000)}); s = writer_->Write(ns, {command}); } @@ -142,7 +142,7 @@ Status Parser::parseComplexKV(const Slice &ns_key, const Metadata &metadata) { } if (metadata.expire > 0) { - output = Redis::Command2RESP({"EXPIREAT", user_key, std::to_string(metadata.expire)}); + output = Redis::Command2RESP({"EXPIREAT", user_key, std::to_string(metadata.expire / 1000)}); Status s = writer_->Write(ns, {output}); if (!s.IsOK()) return s.Prefixed("failed to write the EXPIREAT command to AOF"); } diff --git a/utils/kvrocks2redis/parser.h b/utils/kvrocks2redis/parser.h index 771b9d6ee74..d02df5da535 100644 --- a/utils/kvrocks2redis/parser.h +++ b/utils/kvrocks2redis/parser.h @@ -65,7 +65,7 @@ class Parser { std::unique_ptr latest_snapshot_; bool slot_id_encoded_ = false; - Status parseSimpleKV(const Slice &ns_key, const Slice &value, int expire); + Status parseSimpleKV(const Slice &ns_key, const Slice &value, uint64_t expire); Status parseComplexKV(const Slice &ns_key, const Metadata &metadata); Status parseBitmapSegment(const Slice &ns, const Slice &user_key, int index, const Slice &bitmap); }; From a1b18c891b3f5db0c706cf34a03d1f9db6e64ec6 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 00:11:46 +0800 Subject: [PATCH 02/26] fix --- src/cluster/slot_migrate.cc | 3 +-- src/commands/cmd_key.cc | 8 +++---- src/storage/compact_filter.cc | 2 +- src/storage/redis_db.cc | 12 ++++++----- src/storage/redis_metadata.cc | 30 ++++++++++++++++++++------- src/storage/redis_metadata.h | 6 ++++++ src/types/redis_bitmap_string.cc | 12 ++++++----- src/types/redis_hash.cc | 5 +---- src/types/redis_string.cc | 31 +++++++++++++++++----------- src/types/redis_string.h | 4 +--- tests/cppunit/types/metadata_test.cc | 2 +- utils/kvrocks2redis/parser.cc | 5 +++-- 12 files changed, 74 insertions(+), 46 deletions(-) diff --git a/src/cluster/slot_migrate.cc b/src/cluster/slot_migrate.cc index 467df32777e..c302beb153b 100644 --- a/src/cluster/slot_migrate.cc +++ b/src/cluster/slot_migrate.cc @@ -615,8 +615,7 @@ StatusOr SlotMigrate::MigrateOneKey(const rocksdb::Slice &ke Status SlotMigrate::MigrateSimpleKey(const rocksdb::Slice &key, const Metadata &metadata, const std::string &bytes, std::string *restore_cmds) { - std::vector command = {"set", key.ToString(), - bytes.substr(Redis::STRING_HDR_SIZE, bytes.size() - Redis::STRING_HDR_SIZE)}; + std::vector command = {"set", key.ToString(), bytes.substr(Metadata::GetOffsetAfterExpire(bytes[0]))}; if (metadata.expire > 0) { command.emplace_back("EXAT"); command.emplace_back(std::to_string(metadata.expire / 1000)); diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 30e52292151..dca04f881b0 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -135,7 +135,7 @@ class CommandExpire : public Commander { Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], seconds_); + auto s = redis.Expire(args_[1], seconds_ * 1000); if (s.ok()) { *output = Redis::Integer(1); } else { @@ -157,7 +157,7 @@ class CommandPExpire : public Commander { Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], seconds_); + auto s = redis.Expire(args_[1], seconds_ * 1000); if (s.ok()) { *output = Redis::Integer(1); } else { @@ -189,7 +189,7 @@ class CommandExpireAt : public Commander { Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], timestamp_); + auto s = redis.Expire(args_[1], timestamp_ * 1000); if (s.ok()) { *output = Redis::Integer(1); } else { @@ -221,7 +221,7 @@ class CommandPExpireAt : public Commander { Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], timestamp_); + auto s = redis.Expire(args_[1], timestamp_ * 1000); if (s.ok()) { *output = Redis::Integer(1); } else { diff --git a/src/storage/compact_filter.cc b/src/storage/compact_filter.cc index 812544d8ff7..3325d6f94d6 100644 --- a/src/storage/compact_filter.cc +++ b/src/storage/compact_filter.cc @@ -92,7 +92,7 @@ bool SubKeyFilter::IsMetadataExpired(const InternalKey &ikey, const Metadata &me // // `Util::GetTimeStamp() - 300` means extending 5 minutes for expired items, // to prevent them from being recycled once they reach the expiration time. - int64_t lazy_expired_ts = Util::GetTimeStamp() - 300; + uint64_t lazy_expired_ts = Util::GetTimeStampMS() - 300000; if (metadata.Type() == kRedisString // metadata key was overwrite by set command || metadata.ExpireAt(lazy_expired_ts) || ikey.GetVersion() != metadata.version) { return true; diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 4884bb99672..f9a1710b40a 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -90,16 +90,18 @@ rocksdb::Status Database::Expire(const Slice &user_key, uint64_t timestamp) { if (metadata.Type() != kRedisString && metadata.size == 0) { return rocksdb::Status::NotFound("no elements"); } - if (metadata.expire / 1000 == timestamp) return rocksdb::Status::OK(); + if (metadata.expire == timestamp) return rocksdb::Status::OK(); - auto buf = std::make_unique(value.size()); - memcpy(buf.get(), value.data(), value.size()); // +1 to skip the flags - EncodeFixed32(buf.get() + 1, (uint32_t)timestamp); + if (metadata.Is64BitEncoded()) { + EncodeFixed64(value.data() + 1, timestamp); + } else { + EncodeFixed32(value.data() + 1, timestamp / 1000); + } auto batch = storage_->GetWriteBatchBase(); WriteBatchLogData log_data(kRedisNone, {std::to_string(kRedisCmdExpire)}); batch->PutLogData(log_data.Encode()); - batch->Put(metadata_cf_handle_, ns_key, Slice(buf.get(), value.size())); + batch->Put(metadata_cf_handle_, ns_key, value); s = storage_->Write(storage_->DefaultWriteOptions(), batch->GetWriteBatch()); return s; } diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index 077d6a2661c..f76ab072f2f 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -35,7 +35,7 @@ // 52 bit for microseconds and 11 bit for counter const int VersionCounterBits = 11; -static std::atomic version_counter_ = {0}; +static std::atomic version_counter_ = 0; constexpr const char *kErrMetadataTooShort = "metadata is too short"; @@ -148,7 +148,7 @@ void ComposeSlotKeyPrefix(const Slice &ns, int slotid, std::string *output) { } Metadata::Metadata(RedisType type, bool generate_version) - : flags(uint8_t(0x80) | (uint8_t(0x0f) & type)), expire(0), version(0), size(0) { + : flags(METADATA_64BIT_ENCODING_MASK | (METADATA_TYPE_MASK & type)), expire(0), version(0), size(0) { if (generate_version) version = generateVersion(); } @@ -205,9 +205,25 @@ bool Metadata::operator==(const Metadata &that) const { return true; } -RedisType Metadata::Type() const { return static_cast(flags & uint8_t(0x0f)); } +RedisType Metadata::Type() const { return static_cast(flags & METADATA_TYPE_MASK); } -bool Metadata::Is64BitEncoded() const { return flags & 0x80; } +size_t Metadata::GetOffsetAfterExpire(uint8_t flags) { + if (flags & METADATA_64BIT_ENCODING_MASK) { + return 1 + 8; + } + + return 1 + 4; +} + +size_t Metadata::GetOffsetAfterSize(uint8_t flags) { + if (flags & METADATA_64BIT_ENCODING_MASK) { + return 1 + 8 + 8 + 8; + } + + return 1 + 4 + 8 + 4; +} + +bool Metadata::Is64BitEncoded() const { return flags & METADATA_64BIT_ENCODING_MASK; } size_t Metadata::CommonEncodedSize() const { return Is64BitEncoded() ? 8 : 4; } @@ -275,7 +291,7 @@ bool Metadata::ExpireAt(uint64_t expired_ts) const { return expire < expired_ts; } -bool Metadata::Expired() const { return ExpireAt(Util::GetTimeStamp()); } +bool Metadata::Expired() const { return ExpireAt(Util::GetTimeStampMS()); } ListMetadata::ListMetadata(bool generate_version) : Metadata(kRedisList, generate_version), head(UINT64_MAX / 2), tail(head) {} @@ -289,14 +305,14 @@ void ListMetadata::Encode(std::string *dst) { rocksdb::Status ListMetadata::Decode(const std::string &bytes) { Slice input(bytes); GetFixed8(&input, &flags); - GetFixed32(&input, reinterpret_cast(&expire)); + GetExpire(&input); if (Type() != kRedisString) { if (input.size() < 8 + CommonEncodedSize()) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); GetFixed64(&input, &version); GetFixedCommon(&input, &size); } if (Type() == kRedisList) { - if (input.size() < 16) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); + if (input.size() < 8 + 8) return rocksdb::Status::InvalidArgument(kErrMetadataTooShort); GetFixed64(&input, &head); GetFixed64(&input, &tail); } diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index c1f38efcf32..3dcf39630b1 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -97,6 +97,9 @@ class InternalKey { bool slot_id_encoded_; }; +constexpr uint8_t METADATA_64BIT_ENCODING_MASK = 0x80; +constexpr uint8_t METADATA_TYPE_MASK = 0x0f; + class Metadata { public: uint8_t flags; @@ -107,6 +110,9 @@ class Metadata { explicit Metadata(RedisType type, bool generate_version = true); static void InitVersionCounter(); + static size_t GetOffsetAfterExpire(uint8_t flags); + static size_t GetOffsetAfterSize(uint8_t flags); + bool Is64BitEncoded() const; bool GetFixedCommon(rocksdb::Slice *input, uint64_t *value) const; bool GetExpire(rocksdb::Slice *input); diff --git a/src/types/redis_bitmap_string.cc b/src/types/redis_bitmap_string.cc index d68cd656658..de3a2fac2fc 100644 --- a/src/types/redis_bitmap_string.cc +++ b/src/types/redis_bitmap_string.cc @@ -23,13 +23,14 @@ #include #include "redis_string.h" +#include "storage/redis_metadata.h" namespace Redis { extern const uint8_t kNum2Bits[256]; rocksdb::Status BitmapString::GetBit(const std::string &raw_value, uint32_t offset, bool *bit) { - auto string_value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + auto string_value = raw_value.substr(Metadata::GetOffsetAfterExpire(raw_value[0])); uint32_t byte_index = offset >> 3; uint32_t bit_val = 0; uint32_t bit_offset = 7 - (offset & 0x7); @@ -42,7 +43,8 @@ rocksdb::Status BitmapString::GetBit(const std::string &raw_value, uint32_t offs rocksdb::Status BitmapString::SetBit(const Slice &ns_key, std::string *raw_value, uint32_t offset, bool new_bit, bool *old_bit) { - auto string_value = raw_value->substr(STRING_HDR_SIZE, raw_value->size() - STRING_HDR_SIZE); + size_t header_offset = Metadata::GetOffsetAfterExpire((*raw_value)[0]); + auto string_value = raw_value->substr(header_offset); uint32_t byte_index = offset >> 3; if (byte_index >= string_value.size()) { // expand the bitmap string_value.append(byte_index - string_value.size() + 1, 0); @@ -55,7 +57,7 @@ rocksdb::Status BitmapString::SetBit(const Slice &ns_key, std::string *raw_value byteval = static_cast(byteval | ((new_bit & 0x1) << bit_offset)); string_value[byte_index] = byteval; - *raw_value = raw_value->substr(0, STRING_HDR_SIZE); + *raw_value = raw_value->substr(0, header_offset); raw_value->append(string_value); auto batch = storage_->GetWriteBatchBase(); WriteBatchLogData log_data(kRedisString); @@ -66,7 +68,7 @@ rocksdb::Status BitmapString::SetBit(const Slice &ns_key, std::string *raw_value rocksdb::Status BitmapString::BitCount(const std::string &raw_value, int64_t start, int64_t stop, uint32_t *cnt) { *cnt = 0; - auto string_value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + auto string_value = raw_value.substr(Metadata::GetOffsetAfterExpire(raw_value[0])); /* Convert negative indexes */ if (start < 0 && stop < 0 && start > stop) { return rocksdb::Status::OK(); @@ -89,7 +91,7 @@ rocksdb::Status BitmapString::BitCount(const std::string &raw_value, int64_t sta rocksdb::Status BitmapString::BitPos(const std::string &raw_value, bool bit, int64_t start, int64_t stop, bool stop_given, int64_t *pos) { - auto string_value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + auto string_value = raw_value.substr(Metadata::GetOffsetAfterExpire(raw_value[0])); auto strlen = static_cast(string_value.size()); /* Convert negative indexes */ if (start < 0) start = strlen + start; diff --git a/src/types/redis_hash.cc b/src/types/redis_hash.cc index 2a52fdb10b2..8e914ac1237 100644 --- a/src/types/redis_hash.cc +++ b/src/types/redis_hash.cc @@ -189,10 +189,7 @@ rocksdb::Status Hash::MGet(const Slice &user_key, const std::vector &fiel } rocksdb::Status Hash::Set(const Slice &user_key, const Slice &field, const Slice &value, int *ret) { - FieldValue fv = {field.ToString(), value.ToString()}; - std::vector fvs; - fvs.emplace_back(std::move(fv)); - return MSet(user_key, fvs, false, ret); + return MSet(user_key, {{field.ToString(), value.ToString()}}, false, ret); } rocksdb::Status Hash::Delete(const Slice &user_key, const std::vector &fields, int *ret) { diff --git a/src/types/redis_string.cc b/src/types/redis_string.cc index 79ed41e4ea5..dfc98b1ec0b 100644 --- a/src/types/redis_string.cc +++ b/src/types/redis_string.cc @@ -21,10 +21,13 @@ #include "redis_string.h" #include +#include +#include #include #include #include "parse_util.h" +#include "storage/redis_metadata.h" #include "time_util.h" namespace Redis { @@ -86,7 +89,8 @@ rocksdb::Status String::getValue(const std::string &ns_key, std::string *value) std::string raw_value; auto s = getRawValue(ns_key, &raw_value); if (!s.ok()) return s; - *value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + size_t offset = Metadata::GetOffsetAfterExpire(raw_value[0]); + *value = raw_value.substr(offset); return rocksdb::Status::OK(); } @@ -94,7 +98,8 @@ std::vector String::getValues(const std::vector &ns_keys auto statuses = getRawValues(ns_keys, values); for (size_t i = 0; i < ns_keys.size(); i++) { if (!statuses[i].ok()) continue; - (*values)[i] = (*values)[i].substr(STRING_HDR_SIZE, (*values)[i].size() - STRING_HDR_SIZE); + size_t offset = Metadata::GetOffsetAfterExpire((*values)[i][0]); + (*values)[i] = (*values)[i].substr(offset, (*values)[i].size() - offset); } return statuses; } @@ -121,7 +126,7 @@ rocksdb::Status String::Append(const std::string &user_key, const std::string &v metadata.Encode(&raw_value); } raw_value.append(value); - *ret = static_cast(raw_value.size() - STRING_HDR_SIZE); + *ret = static_cast(raw_value.size() - Metadata::GetOffsetAfterExpire(raw_value[0])); return updateRawValue(ns_key, raw_value); } @@ -240,7 +245,7 @@ rocksdb::Status String::SetXX(const std::string &user_key, const std::string &va return updateRawValue(ns_key, raw_value); } -rocksdb::Status String::SetRange(const std::string &user_key, int offset, const std::string &value, int *ret) { +rocksdb::Status String::SetRange(const std::string &user_key, size_t offset, const std::string &value, int *ret) { std::string ns_key; AppendNamespacePrefix(user_key, &ns_key); @@ -260,11 +265,11 @@ rocksdb::Status String::SetRange(const std::string &user_key, int offset, const metadata.Encode(&raw_value); } - int size = static_cast(raw_value.size()); - offset += STRING_HDR_SIZE; + size_t size = raw_value.size(); + offset += Metadata::GetOffsetAfterExpire(raw_value[0]); if (offset > size) { // padding the value with zero byte while offset is longer than value size - int paddings = offset - size; + size_t paddings = offset - size; raw_value.append(paddings, '\0'); } if (offset + static_cast(value.size()) >= size) { @@ -275,7 +280,7 @@ rocksdb::Status String::SetRange(const std::string &user_key, int offset, const raw_value[offset + i] = value[i]; } } - *ret = static_cast(raw_value.size() - STRING_HDR_SIZE); + *ret = static_cast(raw_value.size() - offset); return updateRawValue(ns_key, raw_value); } @@ -292,7 +297,8 @@ rocksdb::Status String::IncrBy(const std::string &user_key, int64_t increment, i metadata.Encode(&raw_value); } - value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + size_t offset = Metadata::GetOffsetAfterExpire(raw_value[0]); + value = raw_value.substr(offset); int64_t n = 0; if (!value.empty()) { auto parse_result = ParseInt(value, 10); @@ -311,7 +317,7 @@ rocksdb::Status String::IncrBy(const std::string &user_key, int64_t increment, i n += increment; *ret = n; - raw_value = raw_value.substr(0, STRING_HDR_SIZE); + raw_value = raw_value.substr(0, offset); raw_value.append(std::to_string(n)); return updateRawValue(ns_key, raw_value); } @@ -328,7 +334,8 @@ rocksdb::Status String::IncrByFloat(const std::string &user_key, double incremen Metadata metadata(kRedisString, false); metadata.Encode(&raw_value); } - value = raw_value.substr(STRING_HDR_SIZE, raw_value.size() - STRING_HDR_SIZE); + size_t offset = Metadata::GetOffsetAfterExpire(raw_value[0]); + value = raw_value.substr(offset); double n = 0; if (!value.empty()) { auto n_stat = ParseFloat(value); @@ -344,7 +351,7 @@ rocksdb::Status String::IncrByFloat(const std::string &user_key, double incremen } *ret = n; - raw_value = raw_value.substr(0, STRING_HDR_SIZE); + raw_value = raw_value.substr(0, offset); raw_value.append(std::to_string(n)); return updateRawValue(ns_key, raw_value); } diff --git a/src/types/redis_string.h b/src/types/redis_string.h index 98a09c89826..b534545bb32 100644 --- a/src/types/redis_string.h +++ b/src/types/redis_string.h @@ -33,8 +33,6 @@ struct StringPair { namespace Redis { -const int STRING_HDR_SIZE = 5; - class String : public Database { public: explicit String(Engine::Storage *storage, const std::string &ns) : Database(storage, ns) {} @@ -47,7 +45,7 @@ class String : public Database { rocksdb::Status SetEX(const std::string &user_key, const std::string &value, int ttl); rocksdb::Status SetNX(const std::string &user_key, const std::string &value, int ttl, int *ret); rocksdb::Status SetXX(const std::string &user_key, const std::string &value, int ttl, int *ret); - rocksdb::Status SetRange(const std::string &user_key, int offset, const std::string &value, int *ret); + rocksdb::Status SetRange(const std::string &user_key, size_t offset, const std::string &value, int *ret); rocksdb::Status IncrBy(const std::string &user_key, int64_t increment, int64_t *ret); rocksdb::Status IncrByFloat(const std::string &user_key, double increment, double *ret); std::vector MGet(const std::vector &keys, std::vector *values); diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index b82f860db30..830608309ab 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -109,7 +109,7 @@ TEST_F(RedisTypeTest, Expire) { EXPECT_TRUE(s.ok() && static_cast(fvs.size()) == ret); int64_t now = 0; rocksdb::Env::Default()->GetCurrentTime(&now); - redis->Expire(key_, int(now + 2)); + redis->Expire(key_, int(now + 2000)); int64_t ttl = 0; redis->TTL(key_, &ttl); ASSERT_TRUE(ttl >= 1 && ttl <= 2); diff --git a/utils/kvrocks2redis/parser.cc b/utils/kvrocks2redis/parser.cc index e2a32548d03..4a521e9e72e 100644 --- a/utils/kvrocks2redis/parser.cc +++ b/utils/kvrocks2redis/parser.cc @@ -28,6 +28,7 @@ #include "cluster/redis_slot.h" #include "db_util.h" #include "server/redis_reply.h" +#include "storage/redis_metadata.h" #include "types/redis_string.h" Status Parser::ParseFullDB() { @@ -63,8 +64,8 @@ Status Parser::parseSimpleKV(const Slice &ns_key, const Slice &value, uint64_t e std::string ns, user_key; ExtractNamespaceKey(ns_key, &ns, &user_key, slot_id_encoded_); - auto command = Redis::Command2RESP( - {"SET", user_key, value.ToString().substr(Redis::STRING_HDR_SIZE, value.size() - Redis::STRING_HDR_SIZE)}); + auto command = + Redis::Command2RESP({"SET", user_key, value.ToString().substr(Metadata::GetOffsetAfterExpire(value[0]))}); Status s = writer_->Write(ns, {command}); if (!s.IsOK()) return s; From 83daf6e44eff41bd59aa44ef0ef20630ef015c95 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 00:53:51 +0800 Subject: [PATCH 03/26] fix --- src/storage/redis_db.cc | 15 +++++++++------ src/storage/redis_metadata.cc | 11 +++++++++++ tests/cppunit/types/hash_test.cc | 1 + 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index f9a1710b40a..702cefdc7e7 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -30,6 +30,7 @@ #include "rocksdb/iterator.h" #include "server/server.h" #include "storage/redis_metadata.h" +#include "time_util.h" namespace Redis { @@ -156,9 +157,9 @@ rocksdb::Status Database::TTL(const Slice &user_key, int64_t *ttl) { Metadata metadata(kRedisNone, false); metadata.Decode(value); - *ttl = metadata.TTL() / 1000; - if (*ttl < 0) { - *ttl = -2; + *ttl = metadata.TTL(); + if (*ttl > 0) { + *ttl /= 1000; } return rocksdb::Status::OK(); @@ -203,9 +204,11 @@ void Database::Keys(const std::string &prefix, std::vector *keys, K } if (stats) { int64_t ttl = metadata.TTL(); - stats->n_key++; - stats->n_expires++; - if (ttl > 0) ttl_sum += ttl / 1000; + if (ttl != -1) { + stats->n_key++; + stats->n_expires++; + if (ttl > 0) ttl_sum += ttl / 1000; + } } if (keys) { ExtractNamespaceKey(iter->key(), &ns, &user_key, storage_->IsSlotIdEncoded()); diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index f76ab072f2f..0406fbe5353 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -271,7 +271,14 @@ void Metadata::PutExpire(std::string *dst) { } int64_t Metadata::TTL() const { + if (expire == 0) { + return -1; + } + auto now = Util::GetTimeStampMS(); + if (expire < now) { + return -2; + } return int64_t(expire - now); } @@ -288,6 +295,10 @@ bool Metadata::ExpireAt(uint64_t expired_ts) const { if (Type() != kRedisString && Type() != kRedisStream && size == 0) { return true; } + if (expire == 0) { + return false; + } + return expire < expired_ts; } diff --git a/tests/cppunit/types/hash_test.cc b/tests/cppunit/types/hash_test.cc index 2d64b7af3eb..c8a88772716 100644 --- a/tests/cppunit/types/hash_test.cc +++ b/tests/cppunit/types/hash_test.cc @@ -55,6 +55,7 @@ TEST_F(RedisHashTest, GetAndSet) { for (size_t i = 0; i < fields_.size(); i++) { std::string got; auto s = hash->Get(key_, fields_[i], &got); + EXPECT_EQ(s.ToString(), ""); EXPECT_EQ(values_[i], got); } auto s = hash->Delete(key_, fields_, &ret); From 27a66a1e55e1e08420552e6777e75ba4f308069f Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 11:05:22 +0800 Subject: [PATCH 04/26] fix --- src/storage/redis_db.cc | 6 ++---- src/storage/redis_metadata.h | 11 +++++++++++ src/types/redis_string.cc | 23 ++++++++++++----------- tests/cppunit/types/hash_test.cc | 2 +- tests/cppunit/types/metadata_test.cc | 2 +- tests/cppunit/types/string_test.cc | 2 +- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 702cefdc7e7..e712c9014d8 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -157,10 +157,8 @@ rocksdb::Status Database::TTL(const Slice &user_key, int64_t *ttl) { Metadata metadata(kRedisNone, false); metadata.Decode(value); - *ttl = metadata.TTL(); - if (*ttl > 0) { - *ttl /= 1000; - } + auto res = metadata.TTL(); + *ttl = res > 0 ? res / 1000 : res; return rocksdb::Status::OK(); } diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index 3dcf39630b1..c43079a1ef1 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -102,9 +102,20 @@ constexpr uint8_t METADATA_TYPE_MASK = 0x0f; class Metadata { public: + // metadata flags + // <(1-bit) 64bit-common-field-indicator> 0 0 0 <(4-bit) redis-type> + // 64bit-common-field-indicator: make `expire` and `size` 64bit instead of 32bit + // NOTE: `expire` is stored in milliseconds for 64bit, seconds for 32bit + // redis-type: RedisType for the key-value uint8_t flags; + + // expire timestamp, in milliseconds uint64_t expire; + + // the current version: 53bit timestamp + 11bit counter uint64_t version; + + // element size of the key-value uint64_t size; explicit Metadata(RedisType type, bool generate_version = true); diff --git a/src/types/redis_string.cc b/src/types/redis_string.cc index dfc98b1ec0b..4c2f96e6880 100644 --- a/src/types/redis_string.cc +++ b/src/types/redis_string.cc @@ -224,10 +224,10 @@ rocksdb::Status String::SetNX(const std::string &user_key, const std::string &va rocksdb::Status String::SetXX(const std::string &user_key, const std::string &value, int ttl, int *ret) { *ret = 0; int exists = 0; - int expire = 0; + int64_t expire = 0; if (ttl > 0) { int64_t now = Util::GetTimeStamp(); - expire = int(now) + ttl; + expire = now + ttl; } std::string ns_key; @@ -266,13 +266,14 @@ rocksdb::Status String::SetRange(const std::string &user_key, size_t offset, con } size_t size = raw_value.size(); - offset += Metadata::GetOffsetAfterExpire(raw_value[0]); + size_t header_offset = Metadata::GetOffsetAfterExpire(raw_value[0]); + offset += header_offset; if (offset > size) { // padding the value with zero byte while offset is longer than value size size_t paddings = offset - size; raw_value.append(paddings, '\0'); } - if (offset + static_cast(value.size()) >= size) { + if (offset + value.size() >= size) { raw_value = raw_value.substr(0, offset); raw_value.append(value); } else { @@ -280,7 +281,7 @@ rocksdb::Status String::SetRange(const std::string &user_key, size_t offset, con raw_value[offset + i] = value[i]; } } - *ret = static_cast(raw_value.size() - offset); + *ret = static_cast(raw_value.size() - header_offset); return updateRawValue(ns_key, raw_value); } @@ -357,10 +358,10 @@ rocksdb::Status String::IncrByFloat(const std::string &user_key, double incremen } rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { - int expire = 0; + int64_t expire = 0; if (ttl > 0) { int64_t now = Util::GetTimeStamp(); - expire = int(now) + ttl; + expire = now + ttl; } // Data race, key string maybe overwrite by other key while didn't lock the key here, @@ -387,10 +388,10 @@ rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { rocksdb::Status String::MSetNX(const std::vector &pairs, int ttl, int *ret) { *ret = 0; - int expire = 0; + int64_t expire = 0; if (ttl > 0) { int64_t now = Util::GetTimeStamp(); - expire = int(now) + ttl; + expire = now + ttl; } int exists = 0; @@ -452,11 +453,11 @@ rocksdb::Status String::CAS(const std::string &user_key, const std::string &old_ if (old_value == current_value) { std::string raw_value; - int expire = 0; + int64_t expire = 0; Metadata metadata(kRedisString, false); if (ttl > 0) { int64_t now = Util::GetTimeStamp(); - expire = int(now) + ttl; + expire = now + ttl; } metadata.expire = expire * 1000; metadata.Encode(&raw_value); diff --git a/tests/cppunit/types/hash_test.cc b/tests/cppunit/types/hash_test.cc index c8a88772716..4696d7eddc1 100644 --- a/tests/cppunit/types/hash_test.cc +++ b/tests/cppunit/types/hash_test.cc @@ -55,7 +55,7 @@ TEST_F(RedisHashTest, GetAndSet) { for (size_t i = 0; i < fields_.size(); i++) { std::string got; auto s = hash->Get(key_, fields_[i], &got); - EXPECT_EQ(s.ToString(), ""); + EXPECT_EQ(s.ToString(), "OK"); EXPECT_EQ(values_[i], got); } auto s = hash->Delete(key_, fields_, &ret); diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index 830608309ab..52dec8ae3c5 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -109,7 +109,7 @@ TEST_F(RedisTypeTest, Expire) { EXPECT_TRUE(s.ok() && static_cast(fvs.size()) == ret); int64_t now = 0; rocksdb::Env::Default()->GetCurrentTime(&now); - redis->Expire(key_, int(now + 2000)); + redis->Expire(key_, now * 1000 + 2000); int64_t ttl = 0; redis->TTL(key_, &ttl); ASSERT_TRUE(ttl >= 1 && ttl <= 2); diff --git a/tests/cppunit/types/string_test.cc b/tests/cppunit/types/string_test.cc index 52f2de7e765..cdbec94b6d8 100644 --- a/tests/cppunit/types/string_test.cc +++ b/tests/cppunit/types/string_test.cc @@ -143,7 +143,7 @@ TEST_F(RedisStringTest, GetSet) { std::vector values = {"a", "b", "c", "d"}; for (size_t i = 0; i < values.size(); i++) { std::string old_value; - string->Expire(key_, static_cast(now + 1000)); + string->Expire(key_, now * 1000 + 100000); string->GetSet(key_, values[i], &old_value); if (i != 0) { EXPECT_EQ(values[i - 1], old_value); From 0338df03504756a83005a2988bfc43188add5874 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 11:08:52 +0800 Subject: [PATCH 05/26] fix --- src/types/redis_string.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/types/redis_string.cc b/src/types/redis_string.cc index 4c2f96e6880..f2860803628 100644 --- a/src/types/redis_string.cc +++ b/src/types/redis_string.cc @@ -153,10 +153,10 @@ rocksdb::Status String::Get(const std::string &user_key, std::string *value) { } rocksdb::Status String::GetEx(const std::string &user_key, std::string *value, int ttl) { - int expire = 0; + int64_t expire = 0; if (ttl > 0) { int64_t now = Util::GetTimeStamp(); - expire = int(now) + ttl; + expire = now + ttl; } std::string ns_key; AppendNamespacePrefix(user_key, &ns_key); From c1f5cb528d0c00996b4b895fce7ffb4f15e933bc Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 12:36:10 +0800 Subject: [PATCH 06/26] fix --- src/commands/cmd_key.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index dca04f881b0..00fa49ff738 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -145,7 +145,7 @@ class CommandExpire : public Commander { } private: - int seconds_ = 0; + uint64_t seconds_ = 0; }; class CommandPExpire : public Commander { @@ -167,7 +167,7 @@ class CommandPExpire : public Commander { } private: - int seconds_ = 0; + uint64_t seconds_ = 0; }; class CommandExpireAt : public Commander { @@ -199,7 +199,7 @@ class CommandExpireAt : public Commander { } private: - int timestamp_ = 0; + uint64_t timestamp_ = 0; }; class CommandPExpireAt : public Commander { @@ -231,7 +231,7 @@ class CommandPExpireAt : public Commander { } private: - int timestamp_ = 0; + uint64_t timestamp_ = 0; }; class CommandPersist : public Commander { From fbea382b548055db0074cd491ccbe31fd46fbce1 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 12:49:56 +0800 Subject: [PATCH 07/26] fix --- src/storage/redis_db.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index e712c9014d8..63ad203354d 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -205,7 +205,7 @@ void Database::Keys(const std::string &prefix, std::vector *keys, K if (ttl != -1) { stats->n_key++; stats->n_expires++; - if (ttl > 0) ttl_sum += ttl / 1000; + if (ttl > 0) ttl_sum += ttl; } } if (keys) { @@ -224,7 +224,7 @@ void Database::Keys(const std::string &prefix, std::vector *keys, K } if (stats && stats->n_expires > 0) { - stats->avg_ttl = ttl_sum / stats->n_expires; + stats->avg_ttl = ttl_sum / stats->n_expires / 1000; } } From 1e9900a24970e5ba3ce8058bb204d1dfa1ff5420 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 13:27:15 +0800 Subject: [PATCH 08/26] fix --- src/commands/cmd_key.cc | 38 +++++------------------- src/commands/cmd_string.cc | 25 +++++++--------- src/commands/ttl_util.h | 27 +++++------------ src/storage/redis_db.cc | 3 +- src/types/redis_string.cc | 44 ++++++++++++++-------------- src/types/redis_string.h | 17 ++++++----- tests/cppunit/types/metadata_test.cc | 2 +- tests/cppunit/types/string_test.cc | 8 ++--- 8 files changed, 64 insertions(+), 100 deletions(-) diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 00fa49ff738..8a75b2db616 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -74,7 +74,7 @@ class CommandTTL : public Commander { int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (s.ok()) { - *output = Redis::Integer(ttl); + *output = Redis::Integer(ttl > 0 ? ttl / 1000 : ttl); return Status::OK(); } @@ -90,12 +90,7 @@ class CommandPTTL : public Commander { auto s = redis.TTL(args_[1], &ttl); if (!s.ok()) return {Status::RedisExecErr, s.ToString()}; - if (ttl > 0) { - *output = Redis::Integer(ttl * 1000); - } else { - *output = Redis::Integer(ttl); - } - + *output = Redis::Integer(ttl); return Status::OK(); } }; @@ -117,19 +112,10 @@ class CommandExists : public Commander { } }; -StatusOr TTLToTimestamp(int ttl) { - int64_t now = Util::GetTimeStamp(); - if (ttl >= INT32_MAX - now) { - return {Status::RedisParseErr, "the expire time was overflow"}; - } - - return ttl + now; -} - class CommandExpire : public Commander { public: Status Parse(const std::vector &args) override { - seconds_ = GET_OR_RET(TTLToTimestamp(GET_OR_RET(ParseInt(args[2], 10)))); + seconds_ = GET_OR_RET(ParseInt(args[2], 10)) + Util::GetTimeStamp(); return Status::OK(); } @@ -151,13 +137,13 @@ class CommandExpire : public Commander { class CommandPExpire : public Commander { public: Status Parse(const std::vector &args) override { - seconds_ = GET_OR_RET(TTLToTimestamp(TTLMsToS(GET_OR_RET(ParseInt(args[2], 10))))); + seconds_ = GET_OR_RET(ParseInt(args[2], 10)) + Util::GetTimeStampMS(); return Status::OK(); } Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], seconds_ * 1000); + auto s = redis.Expire(args_[1], seconds_); if (s.ok()) { *output = Redis::Integer(1); } else { @@ -178,11 +164,7 @@ class CommandExpireAt : public Commander { return {Status::RedisParseErr, errValueNotInteger}; } - if (*parse_result >= INT32_MAX) { - return {Status::RedisParseErr, "the expire time was overflow"}; - } - - timestamp_ = static_cast(*parse_result); + timestamp_ = *parse_result; return Commander::Parse(args); } @@ -210,18 +192,14 @@ class CommandPExpireAt : public Commander { return {Status::RedisParseErr, errValueNotInteger}; } - if (*parse_result / 1000 >= INT32_MAX) { - return {Status::RedisParseErr, "the expire time was overflow"}; - } - - timestamp_ = static_cast(*parse_result / 1000); + timestamp_ = *parse_result; return Commander::Parse(args); } Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::Database redis(svr->storage_, conn->GetNamespace()); - auto s = redis.Expire(args_[1], timestamp_ * 1000); + auto s = redis.Expire(args_[1], timestamp_); if (s.ok()) { *output = Redis::Integer(1); } else { diff --git a/src/commands/cmd_string.cc b/src/commands/cmd_string.cc index 9184a3d69c2..3b3d56a5875 100644 --- a/src/commands/cmd_string.cc +++ b/src/commands/cmd_string.cc @@ -18,6 +18,7 @@ * */ +#include #include #include "commander.h" @@ -95,7 +96,7 @@ class CommandGetEx : public Commander { } private: - int ttl_ = 0; + uint64_t ttl_ = 0; bool persist_ = false; }; @@ -311,14 +312,14 @@ class CommandSet : public Commander { } private: - int ttl_ = 0; + uint64_t ttl_ = 0; enum { NONE, NX, XX } set_flag_ = NONE; }; class CommandSetEX : public Commander { public: Status Parse(const std::vector &args) override { - auto parse_result = ParseInt(args[2], 10); + auto parse_result = ParseInt(args[2], 10); if (!parse_result) { return {Status::RedisParseErr, errValueNotInteger}; } @@ -332,13 +333,13 @@ class CommandSetEX : public Commander { Status Execute(Server *svr, Connection *conn, std::string *output) override { Redis::String string_db(svr->storage_, conn->GetNamespace()); - auto s = string_db.SetEX(args_[1], args_[3], ttl_); + auto s = string_db.SetEX(args_[1], args_[3], ttl_ * 1000); *output = Redis::SimpleString("OK"); return Status::OK(); } private: - int ttl_ = 0; + uint64_t ttl_ = 0; }; class CommandPSetEX : public Commander { @@ -351,11 +352,7 @@ class CommandPSetEX : public Commander { if (*ttl_ms <= 0) return {Status::RedisParseErr, errInvalidExpireTime}; - if (*ttl_ms < 1000) { - ttl_ = 1; - } else { - ttl_ = static_cast(*ttl_ms / 1000); - } + ttl_ = *ttl_ms; return Commander::Parse(args); } @@ -368,7 +365,7 @@ class CommandPSetEX : public Commander { } private: - int ttl_ = 0; + int64_t ttl_ = 0; }; class CommandMSet : public Commander { @@ -552,9 +549,9 @@ class CommandCAS : public Commander { std::string_view flag; while (parser.Good()) { if (parser.EatEqICaseFlag("EX", flag)) { - ttl_ = GET_OR_RET(parser.TakeInt(TTL_RANGE)); + ttl_ = GET_OR_RET(parser.TakeInt(TTL_RANGE)) * 1000; } else if (parser.EatEqICaseFlag("PX", flag)) { - ttl_ = static_cast(TTLMsToS(GET_OR_RET(parser.TakeInt(TTL_RANGE)))); + ttl_ = GET_OR_RET(parser.TakeInt(TTL_RANGE)); } else { return parser.InvalidSyntax(); } @@ -575,7 +572,7 @@ class CommandCAS : public Commander { } private: - int ttl_ = 0; + uint64_t ttl_ = 0; }; class CommandCAD : public Commander { diff --git a/src/commands/ttl_util.h b/src/commands/ttl_util.h index abed93d299d..457051e5689 100644 --- a/src/commands/ttl_util.h +++ b/src/commands/ttl_util.h @@ -27,35 +27,24 @@ #include "status.h" #include "time_util.h" -template -T TTLMsToS(T ttl) { - if (ttl <= 0) { - return ttl; - } else if (ttl < 1000) { - return 1; - } else { - return ttl / 1000; - } -} - -inline int ExpireToTTL(int64_t expire) { - int64_t now = Util::GetTimeStamp(); - return static_cast(expire - now); +inline uint64_t ExpireToTTL(uint64_t expire) { + uint64_t now = Util::GetTimeStampMS(); + return expire - now; } template constexpr auto TTL_RANGE = NumericRange{1, std::numeric_limits::max()}; template -StatusOr> ParseTTL(CommandParser &parser, std::string_view &curr_flag) { +StatusOr> ParseTTL(CommandParser &parser, std::string_view &curr_flag) { if (parser.EatEqICaseFlag("EX", curr_flag)) { - return GET_OR_RET(parser.template TakeInt(TTL_RANGE)); + return GET_OR_RET(parser.template TakeInt(TTL_RANGE)) * 1000; } else if (parser.EatEqICaseFlag("EXAT", curr_flag)) { - return ExpireToTTL(GET_OR_RET(parser.template TakeInt(TTL_RANGE))); + return ExpireToTTL(GET_OR_RET(parser.template TakeInt(TTL_RANGE)) * 1000); } else if (parser.EatEqICaseFlag("PX", curr_flag)) { - return TTLMsToS(GET_OR_RET(parser.template TakeInt(TTL_RANGE))); + return GET_OR_RET(parser.template TakeInt(TTL_RANGE)); } else if (parser.EatEqICaseFlag("PXAT", curr_flag)) { - return ExpireToTTL(TTLMsToS(GET_OR_RET(parser.template TakeInt(TTL_RANGE)))); + return ExpireToTTL(GET_OR_RET(parser.template TakeInt(TTL_RANGE))); } else { return std::nullopt; } diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 63ad203354d..e43fa69b10a 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -157,8 +157,7 @@ rocksdb::Status Database::TTL(const Slice &user_key, int64_t *ttl) { Metadata metadata(kRedisNone, false); metadata.Decode(value); - auto res = metadata.TTL(); - *ttl = res > 0 ? res / 1000 : res; + *ttl = metadata.TTL(); return rocksdb::Status::OK(); } diff --git a/src/types/redis_string.cc b/src/types/redis_string.cc index f2860803628..edb04c8e736 100644 --- a/src/types/redis_string.cc +++ b/src/types/redis_string.cc @@ -152,10 +152,10 @@ rocksdb::Status String::Get(const std::string &user_key, std::string *value) { return getValue(ns_key, value); } -rocksdb::Status String::GetEx(const std::string &user_key, std::string *value, int ttl) { - int64_t expire = 0; +rocksdb::Status String::GetEx(const std::string &user_key, std::string *value, uint64_t ttl) { + uint64_t expire = 0; if (ttl > 0) { - int64_t now = Util::GetTimeStamp(); + uint64_t now = Util::GetTimeStampMS(); expire = now + ttl; } std::string ns_key; @@ -167,7 +167,7 @@ rocksdb::Status String::GetEx(const std::string &user_key, std::string *value, i std::string raw_data; Metadata metadata(kRedisString, false); - metadata.expire = expire * 1000; + metadata.expire = expire; metadata.Encode(&raw_data); raw_data.append(value->data(), value->size()); auto batch = storage_->GetWriteBatchBase(); @@ -211,22 +211,22 @@ rocksdb::Status String::Set(const std::string &user_key, const std::string &valu return MSet(pairs, 0); } -rocksdb::Status String::SetEX(const std::string &user_key, const std::string &value, int ttl) { +rocksdb::Status String::SetEX(const std::string &user_key, const std::string &value, uint64_t ttl) { std::vector pairs{StringPair{user_key, value}}; return MSet(pairs, ttl); } -rocksdb::Status String::SetNX(const std::string &user_key, const std::string &value, int ttl, int *ret) { +rocksdb::Status String::SetNX(const std::string &user_key, const std::string &value, uint64_t ttl, int *ret) { std::vector pairs{StringPair{user_key, value}}; return MSetNX(pairs, ttl, ret); } -rocksdb::Status String::SetXX(const std::string &user_key, const std::string &value, int ttl, int *ret) { +rocksdb::Status String::SetXX(const std::string &user_key, const std::string &value, uint64_t ttl, int *ret) { *ret = 0; int exists = 0; - int64_t expire = 0; + uint64_t expire = 0; if (ttl > 0) { - int64_t now = Util::GetTimeStamp(); + uint64_t now = Util::GetTimeStampMS(); expire = now + ttl; } @@ -239,7 +239,7 @@ rocksdb::Status String::SetXX(const std::string &user_key, const std::string &va *ret = 1; std::string raw_value; Metadata metadata(kRedisString, false); - metadata.expire = expire * 1000; + metadata.expire = expire; metadata.Encode(&raw_value); raw_value.append(value); return updateRawValue(ns_key, raw_value); @@ -357,10 +357,10 @@ rocksdb::Status String::IncrByFloat(const std::string &user_key, double incremen return updateRawValue(ns_key, raw_value); } -rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { - int64_t expire = 0; +rocksdb::Status String::MSet(const std::vector &pairs, uint64_t ttl) { + uint64_t expire = 0; if (ttl > 0) { - int64_t now = Util::GetTimeStamp(); + uint64_t now = Util::GetTimeStampMS(); expire = now + ttl; } @@ -370,7 +370,7 @@ rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { for (const auto &pair : pairs) { std::string bytes; Metadata metadata(kRedisString, false); - metadata.expire = expire * 1000; + metadata.expire = expire; metadata.Encode(&bytes); bytes.append(pair.value.data(), pair.value.size()); auto batch = storage_->GetWriteBatchBase(); @@ -385,12 +385,12 @@ rocksdb::Status String::MSet(const std::vector &pairs, int ttl) { return rocksdb::Status::OK(); } -rocksdb::Status String::MSetNX(const std::vector &pairs, int ttl, int *ret) { +rocksdb::Status String::MSetNX(const std::vector &pairs, uint64_t ttl, int *ret) { *ret = 0; - int64_t expire = 0; + uint64_t expire = 0; if (ttl > 0) { - int64_t now = Util::GetTimeStamp(); + uint64_t now = Util::GetTimeStampMS(); expire = now + ttl; } @@ -413,7 +413,7 @@ rocksdb::Status String::MSetNX(const std::vector &pairs, int ttl, in } std::string bytes; Metadata metadata(kRedisString, false); - metadata.expire = expire * 1000; + metadata.expire = expire; metadata.Encode(&bytes); bytes.append(pair.value.data(), pair.value.size()); auto batch = storage_->GetWriteBatchBase(); @@ -433,7 +433,7 @@ rocksdb::Status String::MSetNX(const std::vector &pairs, int ttl, in // -1 if the user_key does not exist // 0 if the operation fails rocksdb::Status String::CAS(const std::string &user_key, const std::string &old_value, const std::string &new_value, - int ttl, int *ret) { + uint64_t ttl, int *ret) { *ret = 0; std::string ns_key, current_value; @@ -453,13 +453,13 @@ rocksdb::Status String::CAS(const std::string &user_key, const std::string &old_ if (old_value == current_value) { std::string raw_value; - int64_t expire = 0; + uint64_t expire = 0; Metadata metadata(kRedisString, false); if (ttl > 0) { - int64_t now = Util::GetTimeStamp(); + uint64_t now = Util::GetTimeStampMS(); expire = now + ttl; } - metadata.expire = expire * 1000; + metadata.expire = expire; metadata.Encode(&raw_value); raw_value.append(new_value); auto write_status = updateRawValue(ns_key, raw_value); diff --git a/src/types/redis_string.h b/src/types/redis_string.h index b534545bb32..dcbeda83e55 100644 --- a/src/types/redis_string.h +++ b/src/types/redis_string.h @@ -20,6 +20,7 @@ #pragma once +#include #include #include @@ -38,21 +39,21 @@ class String : public Database { explicit String(Engine::Storage *storage, const std::string &ns) : Database(storage, ns) {} rocksdb::Status Append(const std::string &user_key, const std::string &value, int *ret); rocksdb::Status Get(const std::string &user_key, std::string *value); - rocksdb::Status GetEx(const std::string &user_key, std::string *value, int ttl); + rocksdb::Status GetEx(const std::string &user_key, std::string *value, uint64_t ttl); rocksdb::Status GetSet(const std::string &user_key, const std::string &new_value, std::string *old_value); rocksdb::Status GetDel(const std::string &user_key, std::string *value); rocksdb::Status Set(const std::string &user_key, const std::string &value); - rocksdb::Status SetEX(const std::string &user_key, const std::string &value, int ttl); - rocksdb::Status SetNX(const std::string &user_key, const std::string &value, int ttl, int *ret); - rocksdb::Status SetXX(const std::string &user_key, const std::string &value, int ttl, int *ret); + rocksdb::Status SetEX(const std::string &user_key, const std::string &value, uint64_t ttl); + rocksdb::Status SetNX(const std::string &user_key, const std::string &value, uint64_t ttl, int *ret); + rocksdb::Status SetXX(const std::string &user_key, const std::string &value, uint64_t ttl, int *ret); rocksdb::Status SetRange(const std::string &user_key, size_t offset, const std::string &value, int *ret); rocksdb::Status IncrBy(const std::string &user_key, int64_t increment, int64_t *ret); rocksdb::Status IncrByFloat(const std::string &user_key, double increment, double *ret); std::vector MGet(const std::vector &keys, std::vector *values); - rocksdb::Status MSet(const std::vector &pairs, int ttl = 0); - rocksdb::Status MSetNX(const std::vector &pairs, int ttl, int *ret); - rocksdb::Status CAS(const std::string &user_key, const std::string &old_value, const std::string &new_value, int ttl, - int *ret); + rocksdb::Status MSet(const std::vector &pairs, uint64_t ttl = 0); + rocksdb::Status MSetNX(const std::vector &pairs, uint64_t ttl, int *ret); + rocksdb::Status CAS(const std::string &user_key, const std::string &old_value, const std::string &new_value, + uint64_t ttl, int *ret); rocksdb::Status CAD(const std::string &user_key, const std::string &value, int *ret); private: diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index 52dec8ae3c5..39ac7782c96 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -112,6 +112,6 @@ TEST_F(RedisTypeTest, Expire) { redis->Expire(key_, now * 1000 + 2000); int64_t ttl = 0; redis->TTL(key_, &ttl); - ASSERT_TRUE(ttl >= 1 && ttl <= 2); + ASSERT_TRUE(ttl >= 1000 && ttl <= 2000); sleep(2); } diff --git a/tests/cppunit/types/string_test.cc b/tests/cppunit/types/string_test.cc index cdbec94b6d8..f6a8d8c2f92 100644 --- a/tests/cppunit/types/string_test.cc +++ b/tests/cppunit/types/string_test.cc @@ -179,7 +179,7 @@ TEST_F(RedisStringTest, MSetXX) { EXPECT_EQ(ret, 1); int64_t ttl = 0; string->TTL(key_, &ttl); - EXPECT_TRUE(ttl >= 2 && ttl <= 3); + EXPECT_TRUE(ttl >= 2000 && ttl <= 3000); string->Del(key_); } @@ -213,7 +213,7 @@ TEST_F(RedisStringTest, MSetNXWithTTL) { string->SetNX(key_, "test-value", 3, &ret); int64_t ttl = 0; string->TTL(key_, &ttl); - EXPECT_TRUE(ttl >= 2 && ttl <= 3); + EXPECT_TRUE(ttl >= 2000 && ttl <= 3000); string->Del(key_); } @@ -221,7 +221,7 @@ TEST_F(RedisStringTest, SetEX) { string->SetEX(key_, "test-value", 3); int64_t ttl = 0; string->TTL(key_, &ttl); - EXPECT_TRUE(ttl >= 2 && ttl <= 3); + EXPECT_TRUE(ttl >= 2000 && ttl <= 3000); string->Del(key_); } @@ -276,7 +276,7 @@ TEST_F(RedisStringTest, CAS) { int64_t ttl = 0; string->TTL(key, &ttl); - EXPECT_TRUE(ttl >= 9 && ttl <= 10); + EXPECT_TRUE(ttl >= 9000 && ttl <= 10000); string->Del(key); } From abaa7174d8ad8f0294112c1feb459ab87174f6a1 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 13:34:30 +0800 Subject: [PATCH 09/26] fix --- src/storage/redis_db.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index e43fa69b10a..757a129147a 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -201,8 +201,8 @@ void Database::Keys(const std::string &prefix, std::vector *keys, K } if (stats) { int64_t ttl = metadata.TTL(); + stats->n_key++; if (ttl != -1) { - stats->n_key++; stats->n_expires++; if (ttl > 0) ttl_sum += ttl; } From 847212d0684c8d8c94f552012ab242bfb6f4739d Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 13:39:54 +0800 Subject: [PATCH 10/26] fix --- src/cluster/slot_migrate.cc | 12 +++++------- src/storage/batch_extractor.cc | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/cluster/slot_migrate.cc b/src/cluster/slot_migrate.cc index c302beb153b..23e187188ee 100644 --- a/src/cluster/slot_migrate.cc +++ b/src/cluster/slot_migrate.cc @@ -615,10 +615,10 @@ StatusOr SlotMigrate::MigrateOneKey(const rocksdb::Slice &ke Status SlotMigrate::MigrateSimpleKey(const rocksdb::Slice &key, const Metadata &metadata, const std::string &bytes, std::string *restore_cmds) { - std::vector command = {"set", key.ToString(), bytes.substr(Metadata::GetOffsetAfterExpire(bytes[0]))}; + std::vector command = {"SET", key.ToString(), bytes.substr(Metadata::GetOffsetAfterExpire(bytes[0]))}; if (metadata.expire > 0) { - command.emplace_back("EXAT"); - command.emplace_back(std::to_string(metadata.expire / 1000)); + command.emplace_back("PXAT"); + command.emplace_back(std::to_string(metadata.expire)); } *restore_cmds += Redis::MultiBulkString(command, false); current_pipeline_size_++; @@ -726,8 +726,7 @@ Status SlotMigrate::MigrateComplexKey(const rocksdb::Slice &key, const Metadata // Add TTL for complex key if (metadata.expire > 0) { - *restore_cmds += - Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire / 1000)}, false); + *restore_cmds += Redis::MultiBulkString({"PEXPIREAT", key.ToString(), std::to_string(metadata.expire)}, false); current_pipeline_size_++; } @@ -803,8 +802,7 @@ Status SlotMigrate::MigrateStream(const Slice &key, const StreamMetadata &metada // Add TTL if (metadata.expire > 0) { - *restore_cmds += - Redis::MultiBulkString({"EXPIREAT", key.ToString(), std::to_string(metadata.expire / 1000)}, false); + *restore_cmds += Redis::MultiBulkString({"PEXPIREAT", key.ToString(), std::to_string(metadata.expire)}, false); current_pipeline_size_++; } diff --git a/src/storage/batch_extractor.cc b/src/storage/batch_extractor.cc index 4834a7f54b4..3b4c2f436c1 100644 --- a/src/storage/batch_extractor.cc +++ b/src/storage/batch_extractor.cc @@ -62,7 +62,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic command_args = {"SET", user_key, value.ToString().substr(5, value.size() - 5)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); if (metadata.expire > 0) { - command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire / 1000)}; + command_args = {"PEXPIREAT", user_key, std::to_string(metadata.expire)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); } } else if (metadata.expire > 0) { @@ -74,7 +74,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic } auto cmd = static_cast(*parse_result); if (cmd == kRedisCmdExpire) { - command_args = {"EXPIREAT", user_key, std::to_string(metadata.expire / 1000)}; + command_args = {"PEXPIREAT", user_key, std::to_string(metadata.expire)}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); } } From 94f897452bf549a636fbfc0eaf484ef1155c3b89 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 15:36:47 +0800 Subject: [PATCH 11/26] fix --- tests/cppunit/types/string_test.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/cppunit/types/string_test.cc b/tests/cppunit/types/string_test.cc index f6a8d8c2f92..efd49c1929c 100644 --- a/tests/cppunit/types/string_test.cc +++ b/tests/cppunit/types/string_test.cc @@ -72,6 +72,7 @@ TEST_F(RedisStringTest, MGetAndMSet) { string->MSet(pairs_); std::vector keys; std::vector values; + keys.reserve(pairs_.size()); for (const auto &pair : pairs_) { keys.emplace_back(pair.key); } @@ -172,10 +173,10 @@ TEST_F(RedisStringTest, GetDel) { TEST_F(RedisStringTest, MSetXX) { int ret = 0; - string->SetXX(key_, "test-value", 3, &ret); + string->SetXX(key_, "test-value", 3000, &ret); EXPECT_EQ(ret, 0); string->Set(key_, "test-value"); - string->SetXX(key_, "test-value", 3, &ret); + string->SetXX(key_, "test-value", 3000, &ret); EXPECT_EQ(ret, 1); int64_t ttl = 0; string->TTL(key_, &ttl); @@ -189,6 +190,7 @@ TEST_F(RedisStringTest, MSetNX) { EXPECT_EQ(1, ret); std::vector keys; std::vector values; + keys.reserve(pairs_.size()); for (const auto &pair : pairs_) { keys.emplace_back(pair.key); } @@ -210,7 +212,7 @@ TEST_F(RedisStringTest, MSetNX) { TEST_F(RedisStringTest, MSetNXWithTTL) { int ret = 0; - string->SetNX(key_, "test-value", 3, &ret); + string->SetNX(key_, "test-value", 3000, &ret); int64_t ttl = 0; string->TTL(key_, &ttl); EXPECT_TRUE(ttl >= 2000 && ttl <= 3000); @@ -218,7 +220,7 @@ TEST_F(RedisStringTest, MSetNXWithTTL) { } TEST_F(RedisStringTest, SetEX) { - string->SetEX(key_, "test-value", 3); + string->SetEX(key_, "test-value", 3000); int64_t ttl = 0; string->TTL(key_, &ttl); EXPECT_TRUE(ttl >= 2000 && ttl <= 3000); @@ -257,15 +259,15 @@ TEST_F(RedisStringTest, CAS) { auto status = string->Set(key, value); ASSERT_TRUE(status.ok()); - status = string->CAS("non_exist_key", value, new_value, 10, &ret); + status = string->CAS("non_exist_key", value, new_value, 10000, &ret); ASSERT_TRUE(status.ok()); EXPECT_EQ(-1, ret); - status = string->CAS(key, "cas_value_err", new_value, 10, &ret); + status = string->CAS(key, "cas_value_err", new_value, 10000, &ret); ASSERT_TRUE(status.ok()); EXPECT_EQ(0, ret); - status = string->CAS(key, value, new_value, 10, &ret); + status = string->CAS(key, value, new_value, 10000, &ret); ASSERT_TRUE(status.ok()); EXPECT_EQ(1, ret); From 1f7f60da31aaf7c43f6a35cc417c427cffc89f62 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 16:37:34 +0800 Subject: [PATCH 12/26] fix --- tests/gocase/unit/expire/expire_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/gocase/unit/expire/expire_test.go b/tests/gocase/unit/expire/expire_test.go index 5cc6763caa0..a2f453b7e5b 100644 --- a/tests/gocase/unit/expire/expire_test.go +++ b/tests/gocase/unit/expire/expire_test.go @@ -42,7 +42,7 @@ func TestExpire(t *testing.T) { require.True(t, rdb.Expire(ctx, "x", 5*time.Second).Val()) util.BetweenValues(t, rdb.TTL(ctx, "x").Val(), 4*time.Second, 5*time.Second) require.True(t, rdb.Expire(ctx, "x", 10*time.Second).Val()) - require.Equal(t, 10*time.Second, rdb.TTL(ctx, "x").Val()) + util.BetweenValues(t, rdb.TTL(ctx, "x").Val().Seconds(), 9, 10) require.NoError(t, rdb.Expire(ctx, "x", 2*time.Second).Err()) }) @@ -128,7 +128,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.PExpire(ctx, "x", 100*time.Millisecond).Err()) time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -138,10 +138,10 @@ func TestExpire(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.Set(ctx, "x", "somevalue", 0).Err()) - require.NoError(t, rdb.PExpireAt(ctx, "x", time.UnixMilli(time.Now().Unix()*1000+100)).Err()) - time.Sleep(50 * time.Millisecond) + require.NoError(t, rdb.PExpireAt(ctx, "x", time.UnixMilli(time.Now().Unix()*1000+1500)).Err()) + time.Sleep(800 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(2 * time.Second) + time.Sleep(800 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -153,7 +153,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.Set(ctx, "x", "somevalue", 100*time.Millisecond).Err()) time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(2 * time.Second) + time.Sleep(500 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) From a446a1bc8a80ec3312874837680ae68d44a057a2 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 17:47:04 +0800 Subject: [PATCH 13/26] fix --- src/storage/batch_extractor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/batch_extractor.cc b/src/storage/batch_extractor.cc index 3b4c2f436c1..cd7453ab13c 100644 --- a/src/storage/batch_extractor.cc +++ b/src/storage/batch_extractor.cc @@ -59,7 +59,7 @@ rocksdb::Status WriteBatchExtractor::PutCF(uint32_t column_family_id, const Slic Metadata metadata(kRedisNone); metadata.Decode(value.ToString()); if (metadata.Type() == kRedisString) { - command_args = {"SET", user_key, value.ToString().substr(5, value.size() - 5)}; + command_args = {"SET", user_key, value.ToString().substr(Metadata::GetOffsetAfterExpire(value[0]))}; resp_commands_[ns].emplace_back(Redis::Command2RESP(command_args)); if (metadata.expire > 0) { command_args = {"PEXPIREAT", user_key, std::to_string(metadata.expire)}; From eec9c419a9b71be3fb606790fd0b77f09f1b7b1d Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 18:19:38 +0800 Subject: [PATCH 14/26] fix --- src/storage/redis_metadata.cc | 9 +++++---- src/storage/redis_metadata.h | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index 0406fbe5353..218f4e6feae 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -147,10 +147,11 @@ void ComposeSlotKeyPrefix(const Slice &ns, int slotid, std::string *output) { PutFixed16(output, static_cast(slotid)); } -Metadata::Metadata(RedisType type, bool generate_version) - : flags(METADATA_64BIT_ENCODING_MASK | (METADATA_TYPE_MASK & type)), expire(0), version(0), size(0) { - if (generate_version) version = generateVersion(); -} +Metadata::Metadata(RedisType type, bool generate_version, bool use_64bit_common_field) + : flags((use_64bit_common_field ? METADATA_64BIT_ENCODING_MASK : 0) | (METADATA_TYPE_MASK & type)), + expire(0), + version(generate_version ? generateVersion() : 0), + size(0) {} rocksdb::Status Metadata::Decode(const std::string &bytes) { Slice input(bytes); diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index c43079a1ef1..7df6896b685 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -118,7 +118,7 @@ class Metadata { // element size of the key-value uint64_t size; - explicit Metadata(RedisType type, bool generate_version = true); + explicit Metadata(RedisType type, bool generate_version = true, bool use_64bit_common_field = true); static void InitVersionCounter(); static size_t GetOffsetAfterExpire(uint8_t flags); @@ -132,10 +132,10 @@ class Metadata { RedisType Type() const; size_t CommonEncodedSize() const; - virtual int64_t TTL() const; - virtual timeval Time() const; - virtual bool Expired() const; - virtual bool ExpireAt(uint64_t expired_ts) const; + int64_t TTL() const; + timeval Time() const; + bool Expired() const; + bool ExpireAt(uint64_t expired_ts) const; virtual void Encode(std::string *dst); virtual rocksdb::Status Decode(const std::string &bytes); bool operator==(const Metadata &that) const; From 44ad6a6a35bda613341bda0ddd99c2690a5c61a4 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 18:57:05 +0800 Subject: [PATCH 15/26] disable new encoding by default --- .github/workflows/kvrocks.yaml | 10 +++++++++- CMakeLists.txt | 5 +++++ src/storage/redis_metadata.h | 11 ++++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kvrocks.yaml b/.github/workflows/kvrocks.yaml index fdd173e045f..e98a766f6bb 100644 --- a/.github/workflows/kvrocks.yaml +++ b/.github/workflows/kvrocks.yaml @@ -140,6 +140,14 @@ jobs: os: ubuntu-20.04 without_luajit: -DUSE_LUAJIT=OFF compiler: clang + - name: Ubuntu GCC with new encoding + os: ubuntu-20.04 + compiler: gcc + new_encoding: -DENABLE_NEW_ENCODING=TRUE + - name: Ubuntu Clang with new encoding + os: ubuntu-20.04 + compiler: clang + new_encoding: -DENABLE_NEW_ENCODING=TRUE runs-on: ${{ matrix.os }} steps: @@ -182,7 +190,7 @@ jobs: - name: Build Kvrocks run: | ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.without_jemalloc }} ${{ matrix.without_luajit }} \ - ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} ${{ env.CMAKE_EXTRA_DEFS }} + ${{ matrix.with_ninja }} ${{ matrix.with_sanitizer }} ${{ matrix.with_openssl }} ${{ matrix.new_encoding }} ${{ env.CMAKE_EXTRA_DEFS }} - name: Setup Coredump if: ${{ startsWith(matrix.os, 'ubuntu') }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 30352e3db86..0c522074da1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,8 @@ option(ENABLE_OPENSSL "enable openssl to support tls connection" OFF) option(ENABLE_IPO "enable interprocedural optimization" ON) option(ENABLE_UNWIND "enable libunwind in glog" ON) option(PORTABLE "build a portable binary (disable arch-specific optimizations)" OFF) +# TODO: set ENABLE_NEW_ENCODING to ON when we are ready +option(ENABLE_NEW_ENCODING "enable new encoding (#1033) for storing 64bit size and expire time in milliseconds" OFF) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) @@ -194,6 +196,9 @@ target_link_libraries(kvrocks_objs PUBLIC ${EXTERNAL_LIBS}) if(ENABLE_OPENSSL) target_compile_definitions(kvrocks_objs PUBLIC ENABLE_OPENSSL) endif() +if(ENABLE_NEW_ENCODING) + target_compile_definitions(kvrocks_objs PUBLIC ENABLE_NEW_ENCODING) +endif() if(ENABLE_IPO) include(CheckIPOSupported) diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index 7df6896b685..84dba7f1507 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -29,6 +29,14 @@ #include "encoding.h" #include "types/redis_stream_base.h" +constexpr bool USE_64BIT_COMMON_FIELD_DEFAULT = +#ifdef ENABLE_NEW_ENCODING + true +#else + false +#endif + ; + enum RedisType { kRedisNone, kRedisString, @@ -118,7 +126,8 @@ class Metadata { // element size of the key-value uint64_t size; - explicit Metadata(RedisType type, bool generate_version = true, bool use_64bit_common_field = true); + explicit Metadata(RedisType type, bool generate_version = true, + bool use_64bit_common_field = USE_64BIT_COMMON_FIELD_DEFAULT); static void InitVersionCounter(); static size_t GetOffsetAfterExpire(uint8_t flags); From ba69eabdd3572b7d0b5f1215d4d834b99e84b100 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 20:59:13 +0800 Subject: [PATCH 16/26] fix --- src/commands/ttl_util.h | 9 ++------- src/storage/redis_db.cc | 4 ++-- src/storage/redis_metadata.cc | 14 +++++++++++++- src/storage/redis_metadata.h | 1 + 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/commands/ttl_util.h b/src/commands/ttl_util.h index 457051e5689..64f11a21eaf 100644 --- a/src/commands/ttl_util.h +++ b/src/commands/ttl_util.h @@ -27,11 +27,6 @@ #include "status.h" #include "time_util.h" -inline uint64_t ExpireToTTL(uint64_t expire) { - uint64_t now = Util::GetTimeStampMS(); - return expire - now; -} - template constexpr auto TTL_RANGE = NumericRange{1, std::numeric_limits::max()}; @@ -40,11 +35,11 @@ StatusOr> ParseTTL(CommandParser &parser, std::string_ if (parser.EatEqICaseFlag("EX", curr_flag)) { return GET_OR_RET(parser.template TakeInt(TTL_RANGE)) * 1000; } else if (parser.EatEqICaseFlag("EXAT", curr_flag)) { - return ExpireToTTL(GET_OR_RET(parser.template TakeInt(TTL_RANGE)) * 1000); + return GET_OR_RET(parser.template TakeInt(TTL_RANGE)) * 1000 - Util::GetTimeStampMS(); } else if (parser.EatEqICaseFlag("PX", curr_flag)) { return GET_OR_RET(parser.template TakeInt(TTL_RANGE)); } else if (parser.EatEqICaseFlag("PXAT", curr_flag)) { - return ExpireToTTL(GET_OR_RET(parser.template TakeInt(TTL_RANGE))); + return GET_OR_RET(parser.template TakeInt(TTL_RANGE)) - Util::GetTimeStampMS(); } else { return std::nullopt; } diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 757a129147a..86e0f96ae16 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -97,7 +97,7 @@ rocksdb::Status Database::Expire(const Slice &user_key, uint64_t timestamp) { if (metadata.Is64BitEncoded()) { EncodeFixed64(value.data() + 1, timestamp); } else { - EncodeFixed32(value.data() + 1, timestamp / 1000); + EncodeFixed32(value.data() + 1, Metadata::ExpireMsToS(timestamp)); } auto batch = storage_->GetWriteBatchBase(); WriteBatchLogData log_data(kRedisNone, {std::to_string(kRedisCmdExpire)}); @@ -403,7 +403,7 @@ rocksdb::Status Database::Dump(const Slice &user_key, std::vector * infos->emplace_back("version"); infos->emplace_back(std::to_string(metadata.version)); infos->emplace_back("expire"); - infos->emplace_back(std::to_string(metadata.expire / 1000)); + infos->emplace_back(std::to_string(Metadata::ExpireMsToS(metadata.expire))); infos->emplace_back("size"); infos->emplace_back(std::to_string(metadata.size)); diff --git a/src/storage/redis_metadata.cc b/src/storage/redis_metadata.cc index 218f4e6feae..c6a8c850467 100644 --- a/src/storage/redis_metadata.cc +++ b/src/storage/redis_metadata.cc @@ -224,6 +224,18 @@ size_t Metadata::GetOffsetAfterSize(uint8_t flags) { return 1 + 4 + 8 + 4; } +uint64_t Metadata::ExpireMsToS(uint64_t ms) { + if (ms == 0) { + return 0; + } + + if (ms < 1000) { + return 1; + } + + return ms / 1000; +} + bool Metadata::Is64BitEncoded() const { return flags & METADATA_64BIT_ENCODING_MASK; } size_t Metadata::CommonEncodedSize() const { return Is64BitEncoded() ? 8 : 4; } @@ -267,7 +279,7 @@ void Metadata::PutExpire(std::string *dst) { if (Is64BitEncoded()) { PutFixed64(dst, expire); } else { - PutFixed32(dst, expire / 1000); + PutFixed32(dst, ExpireMsToS(expire)); } } diff --git a/src/storage/redis_metadata.h b/src/storage/redis_metadata.h index 84dba7f1507..062dfce0904 100644 --- a/src/storage/redis_metadata.h +++ b/src/storage/redis_metadata.h @@ -132,6 +132,7 @@ class Metadata { static size_t GetOffsetAfterExpire(uint8_t flags); static size_t GetOffsetAfterSize(uint8_t flags); + static uint64_t ExpireMsToS(uint64_t ms); bool Is64BitEncoded() const; bool GetFixedCommon(rocksdb::Slice *input, uint64_t *value) const; From 8bb08ab4aefe51dec16a5c26e747a0e72c2b6414 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 21:20:13 +0800 Subject: [PATCH 17/26] fix --- tests/gocase/unit/expire/expire_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/gocase/unit/expire/expire_test.go b/tests/gocase/unit/expire/expire_test.go index a2f453b7e5b..b40fe26701f 100644 --- a/tests/gocase/unit/expire/expire_test.go +++ b/tests/gocase/unit/expire/expire_test.go @@ -113,9 +113,9 @@ func TestExpire(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", time.Second).Err()) - time.Sleep(900 * time.Millisecond) + time.Sleep(500 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(1100 * time.Millisecond) + time.Sleep(1500 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -128,7 +128,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.PExpire(ctx, "x", 100*time.Millisecond).Err()) time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(500 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -141,7 +141,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.PExpireAt(ctx, "x", time.UnixMilli(time.Now().Unix()*1000+1500)).Err()) time.Sleep(800 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(800 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -153,7 +153,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.Set(ctx, "x", "somevalue", 100*time.Millisecond).Err()) time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(500 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) From bc4554e52fd7851fc55354e096ad6d3371ce4c33 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 22:17:55 +0800 Subject: [PATCH 18/26] fix --- src/commands/cmd_key.cc | 2 +- tests/gocase/unit/expire/expire_test.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 8a75b2db616..4bd56cfb537 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -74,7 +74,7 @@ class CommandTTL : public Commander { int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (s.ok()) { - *output = Redis::Integer(ttl > 0 ? ttl / 1000 : ttl); + *output = Redis::Integer(Metadata::ExpireMsToS(ttl)); return Status::OK(); } diff --git a/tests/gocase/unit/expire/expire_test.go b/tests/gocase/unit/expire/expire_test.go index b40fe26701f..7cb1ee0d689 100644 --- a/tests/gocase/unit/expire/expire_test.go +++ b/tests/gocase/unit/expire/expire_test.go @@ -112,10 +112,10 @@ func TestExpire(t *testing.T) { t.Run("EXPIRE precision is now the millisecond", func(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) - require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", time.Second).Err()) - time.Sleep(500 * time.Millisecond) + require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", 1500*time.Millisecond).Err()) + time.Sleep(800 * time.Millisecond) a := rdb.Get(ctx, "x").Val() - time.Sleep(1500 * time.Millisecond) + time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() return a == "somevalue" && b == "" }, 3) @@ -125,8 +125,8 @@ func TestExpire(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.Set(ctx, "x", "somevalue", 0).Err()) - require.NoError(t, rdb.PExpire(ctx, "x", 100*time.Millisecond).Err()) - time.Sleep(50 * time.Millisecond) + require.NoError(t, rdb.PExpire(ctx, "x", 1500*time.Millisecond).Err()) + time.Sleep(800 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -150,8 +150,8 @@ func TestExpire(t *testing.T) { t.Run("PSETEX can set sub-second expires", func(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) - require.NoError(t, rdb.Set(ctx, "x", "somevalue", 100*time.Millisecond).Err()) - time.Sleep(50 * time.Millisecond) + require.NoError(t, rdb.Set(ctx, "x", "somevalue", 1500*time.Millisecond).Err()) + time.Sleep(800 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -167,8 +167,8 @@ func TestExpire(t *testing.T) { t.Run("PTTL returns time to live in milliseconds", func(t *testing.T) { require.NoError(t, rdb.Del(ctx, "x").Err()) - require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", 1*time.Second).Err()) - util.BetweenValues(t, rdb.PTTL(ctx, "x").Val(), 900*time.Millisecond, 1000*time.Millisecond) + require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", 2*time.Second).Err()) + util.BetweenValues(t, rdb.PTTL(ctx, "x").Val(), 500*time.Millisecond, 2000*time.Millisecond) }) t.Run("TTL / PTTL return -1 if key has no expire", func(t *testing.T) { From 577593a542bd88947fc547102db6db17538ca259 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 22:20:22 +0800 Subject: [PATCH 19/26] fix --- tests/gocase/unit/expire/expire_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/gocase/unit/expire/expire_test.go b/tests/gocase/unit/expire/expire_test.go index 7cb1ee0d689..7810010f9eb 100644 --- a/tests/gocase/unit/expire/expire_test.go +++ b/tests/gocase/unit/expire/expire_test.go @@ -113,7 +113,7 @@ func TestExpire(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", 1500*time.Millisecond).Err()) - time.Sleep(800 * time.Millisecond) + time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -126,7 +126,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.Set(ctx, "x", "somevalue", 0).Err()) require.NoError(t, rdb.PExpire(ctx, "x", 1500*time.Millisecond).Err()) - time.Sleep(800 * time.Millisecond) + time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -139,7 +139,7 @@ func TestExpire(t *testing.T) { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.Set(ctx, "x", "somevalue", 0).Err()) require.NoError(t, rdb.PExpireAt(ctx, "x", time.UnixMilli(time.Now().Unix()*1000+1500)).Err()) - time.Sleep(800 * time.Millisecond) + time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -151,7 +151,7 @@ func TestExpire(t *testing.T) { util.RetryEventually(t, func() bool { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.Set(ctx, "x", "somevalue", 1500*time.Millisecond).Err()) - time.Sleep(800 * time.Millisecond) + time.Sleep(50 * time.Millisecond) a := rdb.Get(ctx, "x").Val() time.Sleep(2000 * time.Millisecond) b := rdb.Get(ctx, "x").Val() @@ -168,7 +168,7 @@ func TestExpire(t *testing.T) { t.Run("PTTL returns time to live in milliseconds", func(t *testing.T) { require.NoError(t, rdb.Del(ctx, "x").Err()) require.NoError(t, rdb.SetEx(ctx, "x", "somevalue", 2*time.Second).Err()) - util.BetweenValues(t, rdb.PTTL(ctx, "x").Val(), 500*time.Millisecond, 2000*time.Millisecond) + util.BetweenValues(t, rdb.PTTL(ctx, "x").Val(), 50*time.Millisecond, 2000*time.Millisecond) }) t.Run("TTL / PTTL return -1 if key has no expire", func(t *testing.T) { From b09b5c37a17e976b4f8f34961f781fb36ee4f1ba Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 22:33:40 +0800 Subject: [PATCH 20/26] fix --- tests/gocase/unit/expire/expire_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gocase/unit/expire/expire_test.go b/tests/gocase/unit/expire/expire_test.go index 7810010f9eb..d35b84d8801 100644 --- a/tests/gocase/unit/expire/expire_test.go +++ b/tests/gocase/unit/expire/expire_test.go @@ -186,11 +186,11 @@ func TestExpire(t *testing.T) { t.Run("Redis should actively expire keys incrementally", func(t *testing.T) { require.NoError(t, rdb.FlushDB(ctx).Err()) - require.NoError(t, rdb.Do(ctx, "PSETEX", "key1", 500, "a").Err()) - require.NoError(t, rdb.Do(ctx, "PSETEX", "key2", 500, "a").Err()) - require.NoError(t, rdb.Do(ctx, "PSETEX", "key3", 500, "a").Err()) + require.NoError(t, rdb.Do(ctx, "PSETEX", "key1", 1500, "a").Err()) + require.NoError(t, rdb.Do(ctx, "PSETEX", "key2", 1500, "a").Err()) + require.NoError(t, rdb.Do(ctx, "PSETEX", "key3", 1500, "a").Err()) require.NoError(t, rdb.Do(ctx, "DBSIZE", "scan").Err()) - time.Sleep(100 * time.Millisecond) + time.Sleep(50 * time.Millisecond) require.EqualValues(t, 3, rdb.DBSize(ctx).Val()) time.Sleep(2000 * time.Millisecond) require.NoError(t, rdb.Do(ctx, "DBSIZE", "scan").Err()) From d95d75abd282a95b95dab167e16b8687afb507ea Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 22:47:14 +0800 Subject: [PATCH 21/26] fix --- src/commands/cmd_key.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 4bd56cfb537..3302441b7f2 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -74,7 +74,7 @@ class CommandTTL : public Commander { int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (s.ok()) { - *output = Redis::Integer(Metadata::ExpireMsToS(ttl)); + *output = Redis::Integer(ttl > 0 ? Metadata::ExpireMsToS(ttl) : ttl); return Status::OK(); } From 255f2b848e4ef99dd30492c39841075420a8b506 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Wed, 22 Mar 2023 23:02:11 +0800 Subject: [PATCH 22/26] fix --- src/commands/cmd_key.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/cmd_key.cc b/src/commands/cmd_key.cc index 3302441b7f2..8a75b2db616 100644 --- a/src/commands/cmd_key.cc +++ b/src/commands/cmd_key.cc @@ -74,7 +74,7 @@ class CommandTTL : public Commander { int64_t ttl = 0; auto s = redis.TTL(args_[1], &ttl); if (s.ok()) { - *output = Redis::Integer(ttl > 0 ? Metadata::ExpireMsToS(ttl) : ttl); + *output = Redis::Integer(ttl > 0 ? ttl / 1000 : ttl); return Status::OK(); } From 00afbc56fc0e02b8e64c5fb99917f58a84798b72 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Thu, 23 Mar 2023 11:39:11 +0800 Subject: [PATCH 23/26] fix --- src/storage/redis_db.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 86e0f96ae16..c347caa485f 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -404,8 +404,12 @@ rocksdb::Status Database::Dump(const Slice &user_key, std::vector * infos->emplace_back(std::to_string(metadata.version)); infos->emplace_back("expire"); infos->emplace_back(std::to_string(Metadata::ExpireMsToS(metadata.expire))); + infos->emplace_back("pexpire"); + infos->emplace_back(std::to_string(metadata.expire)); infos->emplace_back("size"); infos->emplace_back(std::to_string(metadata.size)); + infos->emplace_back("is_64bit_common_field"); + infos->emplace_back(std::to_string(metadata.Is64BitEncoded())); infos->emplace_back("created_at"); struct timeval created_at = metadata.Time(); From 11f2a80c1fa1fd17e8fa4a2c600a8e34eea1e3e0 Mon Sep 17 00:00:00 2001 From: Twice Date: Thu, 23 Mar 2023 11:52:57 +0800 Subject: [PATCH 24/26] Update src/storage/compact_filter.cc Co-authored-by: hulk --- src/storage/compact_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/compact_filter.cc b/src/storage/compact_filter.cc index 3325d6f94d6..3b4848139e2 100644 --- a/src/storage/compact_filter.cc +++ b/src/storage/compact_filter.cc @@ -90,7 +90,7 @@ bool SubKeyFilter::IsMetadataExpired(const InternalKey &ikey, const Metadata &me // lazy delete to avoid race condition between command Expire and subkey Compaction // Related issue:https://github.com/apache/incubator-kvrocks/issues/1298 // - // `Util::GetTimeStamp() - 300` means extending 5 minutes for expired items, + // `Util::GetTimeStampMS() - 300000` means extending 5 minutes for expired items, // to prevent them from being recycled once they reach the expiration time. uint64_t lazy_expired_ts = Util::GetTimeStampMS() - 300000; if (metadata.Type() == kRedisString // metadata key was overwrite by set command From 558bece6d972a1468405790d38a512a71f5e27e9 Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Thu, 23 Mar 2023 15:07:24 +0800 Subject: [PATCH 25/26] add unit test for compatible Co-authored-by: Yaroslav --- tests/cppunit/types/metadata_test.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index 39ac7782c96..d447da1fea0 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -24,6 +24,7 @@ #include "storage/redis_metadata.h" #include "test_base.h" +#include "time_util.h" #include "types/redis_hash.h" TEST(InternalKey, EncodeAndDecode) { @@ -115,3 +116,18 @@ TEST_F(RedisTypeTest, Expire) { ASSERT_TRUE(ttl >= 1000 && ttl <= 2000); sleep(2); } + +TEST(Metadata, MetadataDecodingBackwardCompatibleSimpleKey) { + auto expire_at = (Util::GetTimeStamp() + 10) * 1000; + Metadata md_old(kRedisString, true, false); + EXPECT_FALSE(md_old.Is64BitEncoded()); + md_old.expire = expire_at; + std::string encoded_bytes; + md_old.Encode(&encoded_bytes); + + Metadata md_new(kRedisNone, false, true); // decoding existing metadata with 64-bit feature activated + md_new.Decode(encoded_bytes); + EXPECT_FALSE(md_new.Is64BitEncoded()); + EXPECT_EQ(md_new.Type(), kRedisString); + EXPECT_EQ(md_new.expire, expire_at); +} From 19e07f8dcd170918dd684f44a307b068c8a8813d Mon Sep 17 00:00:00 2001 From: PragmaTwice Date: Thu, 23 Mar 2023 15:58:52 +0800 Subject: [PATCH 26/26] add more tests Co-authored-by: Yaroslav --- tests/cppunit/types/metadata_test.cc | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/cppunit/types/metadata_test.cc b/tests/cppunit/types/metadata_test.cc index d447da1fea0..68a3515d7fa 100644 --- a/tests/cppunit/types/metadata_test.cc +++ b/tests/cppunit/types/metadata_test.cc @@ -124,6 +124,7 @@ TEST(Metadata, MetadataDecodingBackwardCompatibleSimpleKey) { md_old.expire = expire_at; std::string encoded_bytes; md_old.Encode(&encoded_bytes); + EXPECT_EQ(encoded_bytes.size(), 5); Metadata md_new(kRedisNone, false, true); // decoding existing metadata with 64-bit feature activated md_new.Decode(encoded_bytes); @@ -131,3 +132,61 @@ TEST(Metadata, MetadataDecodingBackwardCompatibleSimpleKey) { EXPECT_EQ(md_new.Type(), kRedisString); EXPECT_EQ(md_new.expire, expire_at); } + +TEST(Metadata, MetadataDecoding64BitSimpleKey) { + auto expire_at = (Util::GetTimeStamp() + 10) * 1000; + Metadata md_old(kRedisString, true, true); + EXPECT_TRUE(md_old.Is64BitEncoded()); + md_old.expire = expire_at; + std::string encoded_bytes; + md_old.Encode(&encoded_bytes); + EXPECT_EQ(encoded_bytes.size(), 9); +} + +TEST(Metadata, MetadataDecodingBackwardCompatibleComplexKey) { + auto expire_at = (Util::GetTimeStamp() + 100) * 1000; + uint32_t size = 1000000000; + Metadata md_old(kRedisHash, true, false); + EXPECT_FALSE(md_old.Is64BitEncoded()); + md_old.expire = expire_at; + md_old.size = size; + std::string encoded_bytes; + md_old.Encode(&encoded_bytes); + + Metadata md_new(kRedisHash, false, true); + md_new.Decode(encoded_bytes); + EXPECT_FALSE(md_new.Is64BitEncoded()); + EXPECT_EQ(md_new.Type(), kRedisHash); + EXPECT_EQ(md_new.expire, expire_at); + EXPECT_EQ(md_new.size, size); +} + +TEST(Metadata, Metadata64bitExpiration) { + auto expire_at = Util::GetTimeStampMS() + 1000; + Metadata md_src(kRedisString, true, true); + EXPECT_TRUE(md_src.Is64BitEncoded()); + md_src.expire = expire_at; + std::string encoded_bytes; + md_src.Encode(&encoded_bytes); + + Metadata md_decoded(kRedisNone, false, true); + md_decoded.Decode(encoded_bytes); + EXPECT_TRUE(md_decoded.Is64BitEncoded()); + EXPECT_EQ(md_decoded.Type(), kRedisString); + EXPECT_EQ(md_decoded.expire, expire_at); +} + +TEST(Metadata, Metadata64bitSize) { + uint64_t big_size = 100000000000; + Metadata md_src(kRedisHash, true, true); + EXPECT_TRUE(md_src.Is64BitEncoded()); + md_src.size = big_size; + std::string encoded_bytes; + md_src.Encode(&encoded_bytes); + + Metadata md_decoded(kRedisNone, false, true); + md_decoded.Decode(encoded_bytes); + EXPECT_TRUE(md_decoded.Is64BitEncoded()); + EXPECT_EQ(md_decoded.Type(), kRedisHash); + EXPECT_EQ(md_decoded.size, big_size); +}