diff --git a/src/cli/main.cc b/src/cli/main.cc index 6e9b0b14a55..32c957a4ed5 100644 --- a/src/cli/main.cc +++ b/src/cli/main.cc @@ -154,12 +154,12 @@ int main(int argc, char *argv[]) { } bool is_supervised = IsSupervisedMode(config.supervised_mode); if (config.daemonize && !is_supervised) Daemonize(); - s = CreatePidFile(config.GetPidFile()); + s = CreatePidFile(config.pidfile); if (!s.IsOK()) { LOG(ERROR) << "Failed to create pidfile: " << s.Msg(); return 1; } - auto pidfile_exit = MakeScopeExit([&config] { RemovePidFile(config.GetPidFile()); }); + auto pidfile_exit = MakeScopeExit([&config] { RemovePidFile(config.pidfile); }); #ifdef ENABLE_OPENSSL // initialize OpenSSL diff --git a/src/commands/cmd_bit.cc b/src/commands/cmd_bit.cc index 088e0add853..541bc38f691 100644 --- a/src/commands/cmd_bit.cc +++ b/src/commands/cmd_bit.cc @@ -22,6 +22,7 @@ #include "commands/command_parser.h" #include "error_constants.h" #include "server/server.h" +#include "status.h" #include "types/redis_bitmap.h" namespace redis { @@ -171,6 +172,10 @@ class CommandBitPos : public Commander { stop_ = *parse_stop; } + if (args.size() >= 6 && util::EqualICase(args[5], "BIT")) { + is_bit_index_ = true; + } + auto parse_arg = ParseInt(args[2], 10); if (!parse_arg) { return {Status::RedisParseErr, errValueNotInteger}; @@ -189,7 +194,7 @@ class CommandBitPos : public Commander { Status Execute(Server *srv, Connection *conn, std::string *output) override { int64_t pos = 0; redis::Bitmap bitmap_db(srv->storage, conn->GetNamespace()); - auto s = bitmap_db.BitPos(args_[1], bit_, start_, stop_, stop_given_, &pos); + auto s = bitmap_db.BitPos(args_[1], bit_, start_, stop_, stop_given_, &pos, is_bit_index_); if (!s.ok()) return {Status::RedisExecErr, s.ToString()}; *output = redis::Integer(pos); @@ -201,6 +206,7 @@ class CommandBitPos : public Commander { int64_t stop_ = -1; bool bit_ = false; bool stop_given_ = false; + bool is_bit_index_ = false; }; class CommandBitOp : public Commander { diff --git a/src/config/config.cc b/src/config/config.cc index 38de6595aba..41408e8b20f 100644 --- a/src/config/config.cc +++ b/src/config/config.cc @@ -42,6 +42,9 @@ #include "status.h" #include "storage/redis_metadata.h" +constexpr const char *kDefaultDir = "/tmp/kvrocks"; +constexpr const char *kDefaultBackupDir = "/tmp/kvrocks/backup"; +constexpr const char *kDefaultPidfile = "/tmp/kvrocks/kvrocks.pid"; constexpr const char *kDefaultBindAddress = "127.0.0.1"; constexpr const char *errBlobDbNotEnabled = "Must set rocksdb.enable_blob_files to yes first."; @@ -141,11 +144,11 @@ Config::Config() { {"force-compact-file-min-deleted-percentage", false, new IntField(&force_compact_file_min_deleted_percentage, 10, 1, 100)}, {"db-name", true, new StringField(&db_name, "change.me.db")}, - {"dir", true, new StringField(&dir, "/tmp/kvrocks")}, - {"backup-dir", false, new StringField(&backup_dir_, "")}, + {"dir", true, new StringField(&dir, kDefaultDir)}, + {"backup-dir", false, new StringField(&backup_dir, kDefaultBackupDir)}, {"log-dir", true, new StringField(&log_dir, "")}, {"log-level", false, new EnumField(&log_level, log_levels, google::INFO)}, - {"pidfile", true, new StringField(&pidfile_, "")}, + {"pidfile", true, new StringField(&pidfile, kDefaultPidfile)}, {"max-io-mb", false, new IntField(&max_io_mb, 0, 0, INT_MAX)}, {"max-bitmap-to-string-mb", false, new IntField(&max_bitmap_to_string_mb, 16, 0, INT_MAX)}, {"max-db-size", false, new IntField(&max_db_size, 0, 0, INT_MAX)}, @@ -403,6 +406,8 @@ void Config::initFieldCallback() { checkpoint_dir = dir + "/checkpoint"; sync_checkpoint_dir = dir + "/sync_checkpoint"; backup_sync_dir = dir + "/backup_for_sync"; + if (backup_dir == kDefaultBackupDir) backup_dir = dir + "/backup"; + if (pidfile == kDefaultPidfile) pidfile = dir + "/kvrocks.pid"; return Status::OK(); }}, {"backup-dir", @@ -412,8 +417,8 @@ void Config::initFieldCallback() { // Note: currently, backup_mu_ may block by backing up or purging, // the command may wait for seconds. std::lock_guard lg(this->backup_mu); - previous_backup = std::move(backup_dir_); - backup_dir_ = v; + previous_backup = std::move(backup_dir); + backup_dir = v; } if (!previous_backup.empty() && srv != nullptr && !srv->IsLoading()) { // LOG(INFO) should be called after log is initialized and server is loaded. @@ -778,9 +783,7 @@ Status Config::finish() { if (master_port != 0 && binds.size() == 0) { return {Status::NotOK, "replication doesn't support unix socket"}; } - if (backup_dir_.empty()) backup_dir_ = dir + "/backup"; if (db_dir.empty()) db_dir = dir + "/db"; - if (pidfile_.empty()) pidfile_ = dir + "/kvrocks.pid"; if (log_dir.empty()) log_dir = dir; std::vector create_dirs = {dir}; for (const auto &name : create_dirs) { diff --git a/src/config/config.h b/src/config/config.h index 2ea2f553e67..fc62b517ca0 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -122,6 +122,8 @@ struct Config { std::vector binds; std::string dir; std::string db_dir; + std::string backup_dir; // GUARD_BY(backup_mu_) + std::string pidfile; std::string backup_sync_dir; std::string checkpoint_dir; std::string sync_checkpoint_dir; @@ -237,13 +239,9 @@ struct Config { void ClearMaster(); bool IsSlave() const { return !master_host.empty(); } bool HasConfigFile() const { return !path_.empty(); } - std::string GetBackupDir() const { return backup_dir_.empty() ? dir + "/backup" : backup_dir_; } - std::string GetPidFile() const { return pidfile_.empty() ? dir + "/kvrocks.pid" : pidfile_; } private: std::string path_; - std::string backup_dir_; // GUARD_BY(backup_mu_) - std::string pidfile_; std::string binds_str_; std::string slaveof_; std::string compact_cron_str_; diff --git a/src/search/ir.h b/src/search/ir.h new file mode 100644 index 00000000000..d431d1bee8b --- /dev/null +++ b/src/search/ir.h @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#pragma once + +#include +#include +#include +#include +#include + +// kqir stands for Kvorcks Query Intermediate Representation +namespace kqir { + +struct FieldRef { + std::string name; +}; + +struct StringLiteral { + std::string val; +}; + +struct TagContainExpr { + FieldRef field; + StringLiteral tag; +}; + +struct NumericLiteral { + double val; +}; + +struct NumericCompareExpr { + enum { EQ, NE, LT, LET, GT, GET } op; + FieldRef field; + NumericLiteral num; +}; + +struct BoolLiteral { + bool val; +}; + +struct BooleanExpr { + std::variant expr; +}; + +struct QueryExpr; + +struct LogicalUnaryExpr { + enum { NOT } op; + std::unique_ptr inner; +}; + +struct LogicalBinaryExpr { + enum { AND, OR } op; + std::unique_ptr lhs; + std::unique_ptr rhs; +}; + +struct QueryExpr { + std::variant expr; +}; + +struct Limit { + size_t offset = 0; + size_t count = std::numeric_limits::max(); +}; + +struct SortBy { + enum { ASC, DESC } order; + FieldRef field; +}; + +struct SelectExpr { + std::vector fields; +}; + +struct IndexRef { + std::string name; +}; + +struct SearchStmt { + IndexRef index_ref; + QueryExpr query_expr; + Limit limit; + SortBy sort_by; + SelectExpr select_expr; +}; + +} // namespace kqir diff --git a/src/storage/storage.cc b/src/storage/storage.cc index f42886b9640..c21b84699e5 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -384,7 +384,7 @@ Status Storage::Open(DBOpenMode mode) { Status Storage::CreateBackup(uint64_t *sequence_number) { LOG(INFO) << "[storage] Start to create new backup"; std::lock_guard lg(config_->backup_mu); - std::string task_backup_dir = config_->GetBackupDir(); + std::string task_backup_dir = config_->backup_dir; std::string tmpdir = task_backup_dir + ".tmp"; // Maybe there is a dirty tmp checkpoint, try to clean it @@ -535,7 +535,7 @@ void Storage::EmptyDB() { void Storage::PurgeOldBackups(uint32_t num_backups_to_keep, uint32_t backup_max_keep_hours) { time_t now = util::GetTimeStamp(); std::lock_guard lg(config_->backup_mu); - std::string task_backup_dir = config_->GetBackupDir(); + std::string task_backup_dir = config_->backup_dir; // Return if there is no backup auto s = env_->FileExists(task_backup_dir); diff --git a/src/types/redis_bitmap.cc b/src/types/redis_bitmap.cc index 555fb13946f..0252420edfb 100644 --- a/src/types/redis_bitmap.cc +++ b/src/types/redis_bitmap.cc @@ -21,7 +21,9 @@ #include "redis_bitmap.h" #include +#include #include +#include #include #include "common/bit_util.h" @@ -307,7 +309,9 @@ rocksdb::Status Bitmap::BitCount(const Slice &user_key, int64_t start, int64_t s } rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, int64_t stop, bool stop_given, - int64_t *pos) { + int64_t *pos, bool is_bit_index) { + if (is_bit_index) DCHECK(stop_given); + std::string raw_value; std::string ns_key = AppendNamespacePrefix(user_key); @@ -323,11 +327,15 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i if (metadata.Type() == kRedisString) { ss = std::nullopt; redis::BitmapString bitmap_string_db(storage_, namespace_); - return bitmap_string_db.BitPos(raw_value, bit, start, stop, stop_given, pos); + return bitmap_string_db.BitPos(raw_value, bit, start, stop, stop_given, pos, is_bit_index); } - std::tie(start, stop) = BitmapString::NormalizeRange(start, stop, static_cast(metadata.size)); - auto u_start = static_cast(start); - auto u_stop = static_cast(stop); + + uint32_t to_bit_factor = is_bit_index ? 8 : 1; + auto size = static_cast(metadata.size) * static_cast(to_bit_factor); + + std::tie(start, stop) = BitmapString::NormalizeRange(start, stop, size); + auto u_start = static_cast(start); + auto u_stop = static_cast(stop); if (u_start > u_stop) { *pos = -1; return rocksdb::Status::OK(); @@ -341,13 +349,40 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i return -1; }; + auto bit_pos_in_byte_startstop = [](char byte, bool bit, uint32_t start, uint32_t stop) -> int { + for (uint32_t i = start; i <= stop; i++) { + if (bit && (byte & (1 << i)) != 0) return (int)i; // typecast to int since the value ranges from 0 to 7 + if (!bit && (byte & (1 << i)) == 0) return (int)i; + } + return -1; + }; + rocksdb::ReadOptions read_options; read_options.snapshot = ss->GetSnapShot(); - uint32_t start_index = u_start / kBitmapSegmentBytes; - uint32_t stop_index = u_stop / kBitmapSegmentBytes; + // if bit index, (Eg start = 1, stop = 35), then + // u_start = 1/8 = 0, u_stop = 35/8 = 4 (in bytes) + uint32_t start_segment_index = (u_start / to_bit_factor) / kBitmapSegmentBytes; + uint32_t stop_segment_index = (u_stop / to_bit_factor) / kBitmapSegmentBytes; + uint32_t start_bit_pos_in_byte = 0; + uint32_t stop_bit_pos_in_byte = 0; + + if (is_bit_index) { + start_bit_pos_in_byte = u_start % 8; + stop_bit_pos_in_byte = u_stop % 8; + } + + auto range_in_byte = [start_bit_pos_in_byte, stop_bit_pos_in_byte]( + uint32_t byte_start, uint32_t byte_end, + uint32_t curr_byte) -> std::pair { + if (curr_byte == byte_start && curr_byte == byte_end) return {start_bit_pos_in_byte, stop_bit_pos_in_byte}; + if (curr_byte == byte_start) return {start_bit_pos_in_byte, 7}; + if (curr_byte == byte_end) return {0, stop_bit_pos_in_byte}; + return {0, 7}; + }; + // Don't use multi get to prevent large range query, and take too much memory // Searching bits in segments [start_index, stop_index]. - for (uint32_t i = start_index; i <= stop_index; i++) { + for (uint32_t i = start_segment_index; i <= stop_segment_index; i++) { rocksdb::PinnableSlice pin_value; std::string sub_key = InternalKey(ns_key, std::to_string(i * kBitmapSegmentBytes), metadata.version, storage_->IsSlotIdEncoded()) @@ -364,17 +399,33 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i continue; } size_t byte_pos_in_segment = 0; - if (i == start_index) byte_pos_in_segment = u_start % kBitmapSegmentBytes; + size_t byte_with_bit_start = -1; + size_t byte_with_bit_stop = -2; + // if bit index, (Eg start = 1, stop = 35), then + // byte_pos_in_segment should be calculated in bytes, hence divide by 8 + if (i == start_segment_index) { + byte_pos_in_segment = (u_start / to_bit_factor) % kBitmapSegmentBytes; + byte_with_bit_start = byte_pos_in_segment; + } size_t stop_byte_in_segment = pin_value.size(); - if (i == stop_index) { - DCHECK_LE(u_stop % kBitmapSegmentBytes + 1, pin_value.size()); - stop_byte_in_segment = u_stop % kBitmapSegmentBytes + 1; + if (i == stop_segment_index) { + DCHECK_LE((u_stop / to_bit_factor) % kBitmapSegmentBytes + 1, pin_value.size()); + stop_byte_in_segment = (u_stop / to_bit_factor) % kBitmapSegmentBytes + 1; + byte_with_bit_stop = stop_byte_in_segment; } // Invariant: // 1. pin_value.size() <= kBitmapSegmentBytes. // 2. If it's the last segment, metadata.size % kBitmapSegmentBytes <= pin_value.size(). for (; byte_pos_in_segment < stop_byte_in_segment; byte_pos_in_segment++) { - int bit_pos_in_byte_value = bit_pos_in_byte(pin_value[byte_pos_in_segment], bit); + int bit_pos_in_byte_value = -1; + if (is_bit_index) { + uint32_t start_bit = 0, stop_bit = 7; + std::tie(start_bit, stop_bit) = range_in_byte(byte_with_bit_start, byte_with_bit_stop, byte_pos_in_segment); + bit_pos_in_byte_value = bit_pos_in_byte_startstop(pin_value[byte_pos_in_segment], bit, start_bit, stop_bit); + } else { + bit_pos_in_byte_value = bit_pos_in_byte(pin_value[byte_pos_in_segment], bit); + } + if (bit_pos_in_byte_value != -1) { *pos = static_cast(i * kBitmapSegmentBits + byte_pos_in_segment * 8 + bit_pos_in_byte_value); return rocksdb::Status::OK(); @@ -387,7 +438,7 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i // 1. If it's the last segment, we've done searching in the above loop. // 2. If it's not the last segment, we can check if the segment is all 0. if (pin_value.size() < kBitmapSegmentBytes) { - if (i == stop_index) { + if (i == stop_segment_index) { continue; } *pos = static_cast(i * kBitmapSegmentBits + pin_value.size() * 8); diff --git a/src/types/redis_bitmap.h b/src/types/redis_bitmap.h index 047a33fd069..33e8e8d23d9 100644 --- a/src/types/redis_bitmap.h +++ b/src/types/redis_bitmap.h @@ -50,7 +50,8 @@ class Bitmap : public Database { rocksdb::Status GetString(const Slice &user_key, uint32_t max_btos_size, std::string *value); rocksdb::Status SetBit(const Slice &user_key, uint32_t bit_offset, bool new_bit, bool *old_bit); rocksdb::Status BitCount(const Slice &user_key, int64_t start, int64_t stop, bool is_bit_index, uint32_t *cnt); - rocksdb::Status BitPos(const Slice &user_key, bool bit, int64_t start, int64_t stop, bool stop_given, int64_t *pos); + rocksdb::Status BitPos(const Slice &user_key, bool bit, int64_t start, int64_t stop, bool stop_given, int64_t *pos, + bool is_bit_index); rocksdb::Status BitOp(BitOpFlags op_flag, const std::string &op_name, const Slice &user_key, const std::vector &op_keys, int64_t *len); rocksdb::Status Bitfield(const Slice &user_key, const std::vector &ops, diff --git a/src/types/redis_bitmap_string.cc b/src/types/redis_bitmap_string.cc index b226d9c2f7a..b10a5d45d49 100644 --- a/src/types/redis_bitmap_string.cc +++ b/src/types/redis_bitmap_string.cc @@ -100,31 +100,80 @@ rocksdb::Status BitmapString::BitCount(const std::string &raw_value, int64_t sta } rocksdb::Status BitmapString::BitPos(const std::string &raw_value, bool bit, int64_t start, int64_t stop, - bool stop_given, int64_t *pos) { + bool stop_given, int64_t *pos, bool is_bit_index) { std::string_view string_value = std::string_view{raw_value}.substr(Metadata::GetOffsetAfterExpire(raw_value[0])); auto strlen = static_cast(string_value.size()); /* Convert negative and out-of-bound indexes */ - std::tie(start, stop) = NormalizeRange(start, stop, strlen); + + int64_t length = is_bit_index ? strlen * 8 : strlen; + std::tie(start, stop) = NormalizeRange(start, stop, length); if (start > stop) { *pos = -1; - } else { - int64_t bytes = stop - start + 1; - *pos = util::msb::RawBitpos(reinterpret_cast(string_value.data()) + start, bytes, bit); - - /* If we are looking for clear bits, and the user specified an exact - * range with start-end, we can't consider the right of the range as - * zero padded (as we do when no explicit end is given). - * - * So if redisBitpos() returns the first bit outside the range, - * we return -1 to the caller, to mean, in the specified range there - * is not a single "0" bit. */ - if (stop_given && bit == 0 && *pos == bytes * 8) { + return rocksdb::Status::OK(); + } + + int64_t byte_start = is_bit_index ? start / 8 : start; + int64_t byte_stop = is_bit_index ? stop / 8 : stop; + int64_t bit_in_start_byte = is_bit_index ? start % 8 : 0; + int64_t bit_in_stop_byte = is_bit_index ? stop % 8 : 7; + int64_t bytes_cnt = byte_stop - byte_start + 1; + + auto bit_pos_in_byte_startstop = [](char byte, bool bit, uint32_t start, uint32_t stop) -> int { + for (uint32_t i = start; i <= stop; i++) { + if (util::msb::GetBitFromByte(byte, i) == bit) { + return (int)i; + } + } + return -1; + }; + + // if the bit start and bit end are in the same byte, we can process it manually + if (is_bit_index && byte_start == byte_stop) { + int res = bit_pos_in_byte_startstop(string_value[byte_start], bit, bit_in_start_byte, bit_in_stop_byte); + if (res != -1) { + *pos = res + byte_start * 8; + return rocksdb::Status::OK(); + } + *pos = -1; + return rocksdb::Status::OK(); + } + + if (is_bit_index && bit_in_start_byte != 0) { + // process first byte + int res = bit_pos_in_byte_startstop(string_value[byte_start], bit, bit_in_start_byte, 7); + if (res != -1) { + *pos = res + byte_start * 8; + return rocksdb::Status::OK(); + } + + byte_start++; + bytes_cnt--; + } + + *pos = util::msb::RawBitpos(reinterpret_cast(string_value.data()) + byte_start, bytes_cnt, bit); + + if (is_bit_index && *pos != -1 && *pos != bytes_cnt * 8) { + // if the pos is more than stop bit, then it is not in the range + if (*pos > stop) { *pos = -1; return rocksdb::Status::OK(); } - if (*pos != -1) *pos += start * 8; /* Adjust for the bytes we skipped. */ } + + /* If we are looking for clear bits, and the user specified an exact + * range with start-end, we tcan' consider the right of the range as + * zero padded (as we do when no explicit end is given). + * + * So if redisBitpos() returns the first bit outside the range, + * we return -1 to the caller, to mean, in the specified range there + * is not a single "0" bit. */ + if (stop_given && bit == 0 && *pos == bytes_cnt * 8) { + *pos = -1; + return rocksdb::Status::OK(); + } + if (*pos != -1) *pos += byte_start * 8; /* Adjust for the bytes we skipped. */ + return rocksdb::Status::OK(); } diff --git a/src/types/redis_bitmap_string.h b/src/types/redis_bitmap_string.h index 7997165afa3..030415c3492 100644 --- a/src/types/redis_bitmap_string.h +++ b/src/types/redis_bitmap_string.h @@ -39,7 +39,7 @@ class BitmapString : public Database { static rocksdb::Status BitCount(const std::string &raw_value, int64_t start, int64_t stop, bool is_bit_index, uint32_t *cnt); static rocksdb::Status BitPos(const std::string &raw_value, bool bit, int64_t start, int64_t stop, bool stop_given, - int64_t *pos); + int64_t *pos, bool is_bit_index); rocksdb::Status Bitfield(const Slice &ns_key, std::string *raw_value, const std::vector &ops, std::vector> *rets); static rocksdb::Status BitfieldReadOnly(const Slice &ns_key, const std::string &raw_value, diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc index 1c28f0bff65..cca20bf1a93 100644 --- a/tests/cppunit/config_test.cc +++ b/tests/cppunit/config_test.cc @@ -175,6 +175,8 @@ TEST(Config, Rewrite) { redis::CommandTable::Reset(); Config config; ASSERT_TRUE(config.Load(CLIOptions(path)).IsOK()); + ASSERT_EQ(config.dir + "/backup", config.backup_dir); + ASSERT_EQ(config.dir + "/kvrocks.pid", config.pidfile); ASSERT_TRUE(config.Rewrite({}).IsOK()); // Need to re-populate the command table since it has renamed by the previous redis::CommandTable::Reset(); diff --git a/tests/cppunit/types/bitmap_test.cc b/tests/cppunit/types/bitmap_test.cc index 4795e476ad0..6ec2d9e39a1 100644 --- a/tests/cppunit/types/bitmap_test.cc +++ b/tests/cppunit/types/bitmap_test.cc @@ -179,7 +179,7 @@ TEST_P(RedisBitmapTest, BitPosClearBit) { /// /// String will set a empty string value when initializing, so, when first /// querying, it should return -1. - bitmap_->BitPos(key_, false, 0, -1, /*stop_given=*/false, &pos); + bitmap_->BitPos(key_, false, 0, -1, /*stop_given=*/false, &pos, /*bit_index=*/false); if (i == 0 && !use_bitmap) { EXPECT_EQ(pos, -1); } else { @@ -201,7 +201,7 @@ TEST_P(RedisBitmapTest, BitPosSetBit) { int64_t pos = 0; int start_indexes[] = {0, 1, 124, 1025, 1027, 3 * 1024 + 1}; for (size_t i = 0; i < sizeof(start_indexes) / sizeof(start_indexes[0]); i++) { - bitmap_->BitPos(key_, true, start_indexes[i], -1, true, &pos); + bitmap_->BitPos(key_, true, start_indexes[i], -1, true, &pos, /*bit_index=*/false); EXPECT_EQ(pos, offsets[i]); } auto s = bitmap_->Del(key_); @@ -215,19 +215,19 @@ TEST_P(RedisBitmapTest, BitPosNegative) { } int64_t pos = 0; // First bit is negative - bitmap_->BitPos(key_, false, 0, -1, true, &pos); + bitmap_->BitPos(key_, false, 0, -1, true, &pos, /*bit_index=*/false); EXPECT_EQ(0, pos); // 8 * 1024 - 1 bit is positive - bitmap_->BitPos(key_, true, 0, -1, true, &pos); + bitmap_->BitPos(key_, true, 0, -1, true, &pos, /*bit_index=*/false); EXPECT_EQ(8 * 1024 - 1, pos); // First bit in 1023 byte is negative - bitmap_->BitPos(key_, false, -1, -1, true, &pos); + bitmap_->BitPos(key_, false, -1, -1, true, &pos, /*bit_index=*/false); EXPECT_EQ(8 * 1023, pos); // Last Bit in 1023 byte is positive - bitmap_->BitPos(key_, true, -1, -1, true, &pos); + bitmap_->BitPos(key_, true, -1, -1, true, &pos, /*bit_index=*/false); EXPECT_EQ(8 * 1024 - 1, pos); // Large negative number will be normalized. - bitmap_->BitPos(key_, false, -10000, -10000, true, &pos); + bitmap_->BitPos(key_, false, -10000, -10000, true, &pos, /*bit_index=*/false); EXPECT_EQ(0, pos); auto s = bitmap_->Del(key_); @@ -242,9 +242,9 @@ TEST_P(RedisBitmapTest, BitPosStopGiven) { EXPECT_FALSE(bit); } int64_t pos = 0; - bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/true, &pos); + bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/true, &pos, /*bit_index=*/false); EXPECT_EQ(-1, pos); - bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/false, &pos); + bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/false, &pos, /*bit_index=*/false); EXPECT_EQ(8, pos); // Set a bit at 8 not affect that @@ -253,9 +253,9 @@ TEST_P(RedisBitmapTest, BitPosStopGiven) { bitmap_->SetBit(key_, 8, true, &bit); EXPECT_FALSE(bit); } - bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/true, &pos); + bitmap_->BitPos(key_, false, 0, 0, /*stop_given=*/true, &pos, /*bit_index=*/false); EXPECT_EQ(-1, pos); - bitmap_->BitPos(key_, false, 0, 1, /*stop_given=*/false, &pos); + bitmap_->BitPos(key_, false, 0, 1, /*stop_given=*/false, &pos, /*bit_index=*/false); EXPECT_EQ(9, pos); auto s = bitmap_->Del(key_); diff --git a/tests/gocase/unit/type/bitmap/bitmap_test.go b/tests/gocase/unit/type/bitmap/bitmap_test.go index 508f52dd82b..55528e30991 100644 --- a/tests/gocase/unit/type/bitmap/bitmap_test.go +++ b/tests/gocase/unit/type/bitmap/bitmap_test.go @@ -378,4 +378,194 @@ func TestBitmap(t *testing.T) { require.EqualValues(t, 't', res.Val()[0]) require.ErrorContains(t, rdb.Do(ctx, "BITFIELD_RO", "str", "INCRBY", "u8", "32", 2).Err(), "BITFIELD_RO only supports the GET subcommand") }) + + t.Run("BITPOS BIT option check", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "mykey", "\x00\xff\xf0", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "mykey", 1, 7, 15, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 8, cmd.Val()) + }) + + t.Run("BITPOS BIT not found check check", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "mykey", "\x00\xff\xf0", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "mykey", 0, 0, 5, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 0, cmd.Val()) + }) + + t.Run("BITPOS BIT not found check check", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "mykey", "\x00\xff\xf0", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "mykey", 0, 2, 3, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 2, cmd.Val()) + }) + + /* Test cases adapted from redis test cases : https://github.com/redis/redis/blob/unstable/tests/unit/bitops.tcl + */ + t.Run("BITPOS bit=0 with empty key returns 0", func(t *testing.T) { + require.NoError(t, rdb.Del(ctx, "str").Err()) + cmd := rdb.BitPosSpan(ctx, "str", 0, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 0, cmd.Val()) + }) + + t.Run("BITPOS bit=0 with string less than 1 word works", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\xff\xf0\x00", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 0, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 12, cmd.Val()) + }) + + t.Run("BITPOS bit=1 with string less than 1 word works", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\x00\x0f\x00", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 1, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 12, cmd.Val()) + }) + + t.Run("BITPOS bit=0 starting at unaligned address", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\xff\xf0\x00", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 0, 1, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 12, cmd.Val()) + }) + + t.Run("BITPOS bit=1 starting at unaligned address", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\x00\x0f\xff", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 1, 1, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 12, cmd.Val()) + }) + + t.Run("BITPOS bit=0 unaligned+full word+reminder", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\xff\xff\xff", 0).Err()) + require.NoError(t, rdb.Append(ctx, "str", "\xff\xff\xff\xff\xff\xff\xff\xff").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\xff\xff\xff\xff\xff\xff\xff\xff").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\xff\xff\xff\xff\xff\xff\xff\xff").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\x0f").Err()) + // Test values 1, 9, 17, 25, 33, 41, 49, 57, 65 + for i := 0; i < 9; i++ { + if i == 6 { + cmd := rdb.BitPosSpan(ctx, "str", 0, 41, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 216, cmd.Val()) + } else { + cmd := rdb.BitPosSpan(ctx, "str", 0, int64(i*8)+1, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 216, cmd.Val()) + } + } + }) + + t.Run("BITPOS bit=1 unaligned+full word+reminder", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\x00\x00\x00", 0).Err()) + require.NoError(t, rdb.Append(ctx, "str", "\x00\x00\x00\x00\x00\x00\x00\x00").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\x00\x00\x00\x00\x00\x00\x00\x00").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\x00\x00\x00\x00\x00\x00\x00\x00").Err()) + require.NoError(t, rdb.Append(ctx, "str", "\xf0").Err()) + // Test values 1, 9, 17, 25, 33, 41, 49, 57, 65 + for i := 0; i < 9; i++ { + if i == 6 { + cmd := rdb.BitPosSpan(ctx, "str", 1, 41, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 216, cmd.Val()) + } else { + cmd := rdb.BitPosSpan(ctx, "str", 1, int64(i*8)+1, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 216, cmd.Val()) + } + } + }) + + t.Run("BITPOS bit=1 returns -1 if string is all 0 bits", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "", 0).Err()) + for i := 0; i < 20; i++ { + cmd := rdb.BitPosSpan(ctx, "str", 1, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, -1, cmd.Val()) + require.NoError(t, rdb.Append(ctx, "str", "\x00").Err()) + } + }) + + t.Run("BITPOS bit=0 works with intervals", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\x00\xff\x00", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 0, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 0, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 0, 8, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 16, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 0, 16, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 16, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 0, 16, 200, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 16, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 0, 8, 8, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, -1, cmd.Val()) + }) + + t.Run("BITPOS bit=1 works with intervals", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\x00\xff\x00", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 1, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 8, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 1, 8, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 8, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 1, 16, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, -1, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 1, 16, 200, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, -1, cmd.Val()) + cmd = rdb.BitPosSpan(ctx, "str", 1, 8, 8, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, 8, cmd.Val()) + }) + + t.Run("BITPOS bit=0 changes behavior if end is given", func(t *testing.T) { + require.NoError(t, rdb.Set(ctx, "str", "\xff\xff\xff", 0).Err()) + cmd := rdb.BitPosSpan(ctx, "str", 0, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, -1, cmd.Val()) + }) + + t.Run("BITPOS bit=1 fuzzy testing using SETBIT", func(t *testing.T) { + require.NoError(t, rdb.Del(ctx, "str").Err()) + var max int64 = 524288 + var firstOnePos int64 = -1 + for j := 0; j < 1000; j++ { + cmd := rdb.BitPosSpan(ctx, "str", 1, 0, -1, "bit") + require.NoError(t, cmd.Err()) + require.EqualValues(t, firstOnePos, cmd.Val()) + pos := util.RandomInt(max) + require.NoError(t, rdb.SetBit(ctx, "str", int64(pos), 1).Err()) + if firstOnePos == -1 || firstOnePos > pos { + firstOnePos = pos + } + } + }) + + t.Run("BITPOS bit=0 fuzzy testing using SETBIT", func(t *testing.T) { + var max int64 = 524288 + firstZeroPos := max + require.NoError(t, rdb.Set(ctx, "str", strings.Repeat("\xff", int(max/8)), 0).Err()) + for j := 0; j < 1000; j++ { + cmd := rdb.BitPosSpan(ctx, "str", 0, 0, -1, "bit") + require.NoError(t, cmd.Err()) + if firstZeroPos == max { + require.EqualValues(t, -1, cmd.Val()) + } else { + require.EqualValues(t, firstZeroPos, cmd.Val()) + } + pos := util.RandomInt(max) + require.NoError(t, rdb.SetBit(ctx, "str", int64(pos), 0).Err()) + if firstZeroPos > pos { + firstZeroPos = pos + } + } + }) + }