Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add unchanged names to existence filter and watchFilters spec test runner #10862

Merged
merged 18 commits into from
Mar 18, 2023
Merged

Add unchanged names to existence filter and watchFilters spec test runner #10862

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
969f2ed
add unchanged names to existence filter and modify spec test runner
milaGGL Feb 24, 2023
b66b02b
Merge branch 'mila/BloomFilter' into mila/BloomFilter-add--existence-…
milaGGL Feb 28, 2023
1df9410
use BloomFilter instead of proto
milaGGL Feb 28, 2023
1e67033
update serializer
milaGGL Feb 28, 2023
899e3cc
add bloomfilter equality test
milaGGL Mar 3, 2023
3faa3e8
Merge branch 'mila/BloomFilter' into mila/BloomFilter-add--existence-…
milaGGL Mar 6, 2023
468f967
make unchanged_names optional in proto
milaGGL Mar 7, 2023
23f0fb1
handle invalid unchanged-names input with BloomFilter::Create
milaGGL Mar 7, 2023
475fffb
format the comments
milaGGL Mar 7, 2023
5bda231
Merge branch 'mila/BloomFilter' into mila/BloomFilter-add--existence-…
milaGGL Mar 13, 2023
05caaeb
change unchanged_names to Bloom_filter
milaGGL Mar 13, 2023
d56f4df
remove the parsing unchanged_names to BloomFilter logic
milaGGL Mar 14, 2023
91a42e6
resolve comments
milaGGL Mar 16, 2023
e694551
Merge branch 'mila/BloomFilter' into mila/BloomFilter-add--existence-…
milaGGL Mar 16, 2023
a2b7408
format
milaGGL Mar 16, 2023
2a71c20
resolve comments
milaGGL Mar 17, 2023
4a97630
resolve comments
milaGGL Mar 17, 2023
3eec288
Update bloom_filter.cc
milaGGL Mar 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 30 additions & 24 deletions Firestore/core/src/remote/bloom_filter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,35 @@ namespace remote {
using util::Status;
using util::StatusOr;

namespace {
bool HasSameBits(const BloomFilter& lhs, const BloomFilter& rhs) {
if (lhs.bit_count() != rhs.bit_count()) {
return false;
}
if (lhs.bit_count() == 0) {
return true;
}

const std::vector<uint8_t>& bitmap1 = lhs.bitmap();
const std::vector<uint8_t>& bitmap2 = rhs.bitmap();
const auto byte_count = static_cast<int32_t>(bitmap1.size());

// Compare all bytes from the bitmap, except for the last byte.
for (int32_t i = 0; i < byte_count - 1; ++i) {
if (bitmap1[i] != bitmap2[i]) {
return false;
}
}

// Compare the last byte, ignoring the padding.
const int32_t padding = (byte_count * 8) - lhs.bit_count();
const uint8_t last_byte1 = bitmap1[byte_count - 1] << padding;
const uint8_t last_byte2 = bitmap2[byte_count - 1] << padding;

return (last_byte1 == last_byte2);
}
} // namespace

BloomFilter::Hash BloomFilter::Md5HashDigest(absl::string_view key) const {
std::array<uint8_t, 16> md5_digest{util::CalculateMd5Digest(key)};

Expand Down Expand Up @@ -115,30 +144,7 @@ bool BloomFilter::MightContain(absl::string_view value) const {

bool operator==(const BloomFilter& lhs, const BloomFilter& rhs) {
dconeybe marked this conversation as resolved.
Show resolved Hide resolved
return lhs.bit_count() == rhs.bit_count() &&
lhs.hash_count() == rhs.hash_count() && CompareBits(lhs, rhs);
}

bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) {
return !(lhs == rhs);
}

bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs) {
if (lhs.bit_count() != rhs.bit_count()) {
return false;
}

for (int32_t i = 0; i < lhs.bit_count(); ++i) {
int32_t offset = i % 8;
const uint8_t& lhs_byte = lhs.bitmap()[i / 8];
const uint8_t& rhs_byte = rhs.bitmap()[i / 8];
const bool bit1 = (lhs_byte & (static_cast<uint8_t>(0x01) << offset));
const bool bit2 = (rhs_byte & (static_cast<uint8_t>(0x01) << offset));
if (bit1 != bit2) {
return false;
}
}

return true;
lhs.hash_count() == rhs.hash_count() && HasSameBits(lhs, rhs);
dconeybe marked this conversation as resolved.
Show resolved Hide resolved
}

} // namespace remote
Expand Down
6 changes: 3 additions & 3 deletions Firestore/core/src/remote/bloom_filter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ class BloomFilter final {

bool operator==(const BloomFilter& lhs, const BloomFilter& rhs);
dconeybe marked this conversation as resolved.
Show resolved Hide resolved

bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs);

bool CompareBits(const BloomFilter& lhs, const BloomFilter& rhs);
inline bool operator!=(const BloomFilter& lhs, const BloomFilter& rhs) {
return !(lhs == rhs);
};

} // namespace remote
} // namespace firestore
Expand Down
38 changes: 28 additions & 10 deletions Firestore/core/test/unit/remote/bloom_filter_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,18 @@ TEST(BloomFilterTest, CheckBloomFiltersEqualityWithSameInput) {
}

TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentBitmap) {
dconeybe marked this conversation as resolved.
Show resolved Hide resolved
BloomFilter bloom_filter1(std::vector<uint8_t>{1}, 1, 1);
BloomFilter bloom_filter2(std::vector<uint8_t>{2}, 1, 1);
EXPECT_FALSE(bloom_filter1 == bloom_filter2);
EXPECT_TRUE(bloom_filter1 != bloom_filter2);
{
BloomFilter bloom_filter1(std::vector<uint8_t>{1}, 1, 1);
BloomFilter bloom_filter2(std::vector<uint8_t>{2}, 1, 1);
EXPECT_FALSE(bloom_filter1 == bloom_filter2);
EXPECT_TRUE(bloom_filter1 != bloom_filter2);
}
{
BloomFilter bloom_filter1(std::vector<uint8_t>{1}, 1, 1);
dconeybe marked this conversation as resolved.
Show resolved Hide resolved
BloomFilter bloom_filter2(std::vector<uint8_t>{1, 1}, 1, 1);
EXPECT_FALSE(bloom_filter1 == bloom_filter2);
EXPECT_TRUE(bloom_filter1 != bloom_filter2);
}
}

TEST(BloomFilterTest, CheckBloomFiltersEqualityWithDifferentPadding) {
Expand All @@ -150,12 +158,22 @@ TEST(BloomFilterTest,
BloomFiltersEqualityCheckShouldIgnoreBitsInPaddingIndexes) {
// In BloomFilter bitmap, padding is guaranteed to be less than 8, and
// starts counting from the leftmost indexes of the last byte.
BloomFilter bloom_filter1(std::vector<uint8_t>{255, 127}, 1,
1); // bitmap -> 11111111 01111111
BloomFilter bloom_filter2(std::vector<uint8_t>{255, 255}, 1,
1); // bitmap -> 11111111 11111111
EXPECT_TRUE(bloom_filter1 == bloom_filter2);
EXPECT_FALSE(bloom_filter1 != bloom_filter2);
{
BloomFilter bloom_filter1(std::vector<uint8_t>{255, 127}, 1,
1); // bitmap -> 11111111 01111111
BloomFilter bloom_filter2(std::vector<uint8_t>{255, 255}, 1,
1); // bitmap -> 11111111 11111111
EXPECT_TRUE(bloom_filter1 == bloom_filter2);
EXPECT_FALSE(bloom_filter1 != bloom_filter2);
}
{
BloomFilter bloom_filter1(std::vector<uint8_t>{255, 207}, 4,
1); // bitmap -> 11111111 11001111
BloomFilter bloom_filter2(std::vector<uint8_t>{255, 255}, 4,
1); // bitmap -> 11111111 11111111
EXPECT_TRUE(bloom_filter1 == bloom_filter2);
EXPECT_FALSE(bloom_filter1 != bloom_filter2);
}
}

TEST(BloomFilterTest, MightContainCanProcessNonStandardCharacters) {
Expand Down