Skip to content

Commit

Permalink
pw_kvs: Support multiple entry formats
Browse files Browse the repository at this point in the history 
Change-Id: If25ea00210178b8495585a651d3df6ad8347ee82
  • Loading branch information
@255
255 committed Mar 6, 2020
1 parent 8e94ed6 commit 22d0d9f
Show file tree
Hide file tree
Showing 12 changed files with 299 additions and 109 deletions.
1 change: 1 addition & 0 deletions pw_kvs/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ pw_cc_library(
"checksum.cc",
"entry.cc",
"flash_memory.cc",
"format.cc",
"key_value_store.cc",
"public/pw_kvs/internal/entry.h",
"public/pw_kvs/internal/hash.h",
Expand Down
1 change: 1 addition & 0 deletions pw_kvs/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ source_set("pw_kvs") {
"checksum.cc",
"entry.cc",
"flash_memory.cc",
"format.cc",
"key_value_store.cc",
"public/pw_kvs/internal/entry.h",
"public/pw_kvs/internal/hash.h",
Expand Down
57 changes: 33 additions & 24 deletions pw_kvs/entry.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ namespace pw::kvs::internal {
using std::byte;
using std::string_view;

Status Entry::Read(FlashPartition& partition, Address address, Entry* entry) {
Status Entry::Read(FlashPartition& partition,
Address address,
const internal::EntryFormats& formats,
Entry* entry) {
EntryHeader header;
TRY(partition.Read(address, sizeof(header), &header));

Expand All @@ -36,7 +39,15 @@ Status Entry::Read(FlashPartition& partition, Address address, Entry* entry) {
return Status::DATA_LOSS;
}

*entry = Entry(&partition, address, header);
const EntryFormat* format = formats.Find(header.magic);
if (format == nullptr) {
PW_LOG_ERROR("Found corrupt magic: %" PRIx32 " at address %zx",
header.magic,
size_t(address));
return Status::DATA_LOSS;
}

*entry = Entry(&partition, address, *format, header);
return Status::OK;
}

Expand All @@ -61,15 +72,16 @@ Entry::Entry(FlashPartition& partition,
uint32_t transaction_id)
: Entry(&partition,
address,
format,
{.magic = format.magic,
.checksum = 0,
.alignment_units =
alignment_bytes_to_units(partition.alignment_bytes()),
.key_length_bytes = static_cast<uint8_t>(key.size()),
.value_size_bytes = value_size_bytes,
.transaction_id = transaction_id}) {
if (format.checksum != nullptr) {
span<const byte> checksum = CalculateChecksum(format.checksum, key, value);
if (checksum_ != nullptr) {
span<const byte> checksum = CalculateChecksum(key, value);
std::memcpy(&header_.checksum,
checksum.data(),
std::min(checksum.size(), sizeof(header_.checksum)));
Expand Down Expand Up @@ -104,17 +116,15 @@ StatusWithSize Entry::ReadValue(span<byte> buffer, size_t offset_bytes) const {
return StatusWithSize(read_size);
}

Status Entry::VerifyChecksum(ChecksumAlgorithm* algorithm,
string_view key,
span<const byte> value) const {
if (algorithm == nullptr) {
Status Entry::VerifyChecksum(string_view key, span<const byte> value) const {
if (checksum_ == nullptr) {
return checksum() == 0 ? Status::OK : Status::DATA_LOSS;
}
CalculateChecksum(algorithm, key, value);
return algorithm->Verify(checksum_bytes());
CalculateChecksum(key, value);
return checksum_->Verify(checksum_bytes());
}

Status Entry::VerifyChecksumInFlash(ChecksumAlgorithm* algorithm) const {
Status Entry::VerifyChecksumInFlash() const {
// Read the entire entry piece-by-piece into a small buffer. If the entry is
// 32 B or less, only one read is required.
union {
Expand All @@ -137,18 +147,18 @@ Status Entry::VerifyChecksumInFlash(ChecksumAlgorithm* algorithm) const {
return Status::DATA_LOSS;
}

if (algorithm == nullptr) {
if (checksum_ == nullptr) {
return checksum() == 0 ? Status::OK : Status::DATA_LOSS;
}

// The checksum is calculated as if the header's checksum field were 0.
header_to_verify.checksum = 0;

algorithm->Reset();
checksum_->Reset();

while (true) {
// Add the chunk in the buffer to the checksum.
algorithm->Update(buffer, read_size);
checksum_->Update(buffer, read_size);

bytes_to_read -= read_size;
if (bytes_to_read == 0u) {
Expand All @@ -161,8 +171,8 @@ Status Entry::VerifyChecksumInFlash(ChecksumAlgorithm* algorithm) const {
TRY(partition().Read(read_address, read_size, buffer));
}

algorithm->Finish();
return algorithm->Verify(checksum_bytes());
checksum_->Finish();
return checksum_->Verify(checksum_bytes());
}

void Entry::DebugLog() {
Expand All @@ -176,18 +186,17 @@ void Entry::DebugLog() {
PW_LOG_DEBUG(" Alignment = 0x%zx", size_t(alignment_bytes()));
}

span<const byte> Entry::CalculateChecksum(ChecksumAlgorithm* algorithm,
const string_view key,
span<const byte> Entry::CalculateChecksum(const string_view key,
span<const byte> value) const {
algorithm->Reset();
checksum_->Reset();

{
EntryHeader header_for_checksum = header_;
header_for_checksum.checksum = 0;

algorithm->Update(&header_for_checksum, sizeof(header_for_checksum));
algorithm->Update(as_bytes(span(key)));
algorithm->Update(value);
checksum_->Update(&header_for_checksum, sizeof(header_for_checksum));
checksum_->Update(as_bytes(span(key)));
checksum_->Update(value);
}

// Update the checksum with 0s to pad the entry to its alignment boundary.
Expand All @@ -196,11 +205,11 @@ span<const byte> Entry::CalculateChecksum(ChecksumAlgorithm* algorithm,

while (padding_to_add > 0u) {
const size_t chunk_size = std::min(padding_to_add, sizeof(padding));
algorithm->Update(padding, chunk_size);
checksum_->Update(padding, chunk_size);
padding_to_add -= chunk_size;
}

return algorithm->Finish();
return checksum_->Finish();
}

} // namespace pw::kvs::internal
62 changes: 34 additions & 28 deletions pw_kvs/entry_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ TEST(Entry, Size_RoundsUpToAlignment) {
for (size_t value : {size_t(0), align - 1, align, align + 1, 2 * align}) {
Entry entry =
Entry::Valid(partition, 0, kFormat, "k", {nullptr, value}, 0);

ASSERT_EQ(AlignUp(sizeof(EntryHeader) + 1 /* key */ + value, align),
entry.size());
}
Expand Down Expand Up @@ -77,8 +78,10 @@ TEST(Entry, Construct_Tombstone) {
EXPECT_EQ(entry.transaction_id(), 123u);
}

constexpr uint32_t kMagicWithChecksum = 0x600df00d;

constexpr auto kHeader1 = ByteStr(
"\x0d\xf0\x0d\x60" // magic
"\x0d\xf0\x0d\x60" // magic (600df00d)
"\xC5\x65\x00\x00" // checksum (CRC16)
"\x01" // alignment
"\x05" // key length
Expand All @@ -93,10 +96,14 @@ constexpr auto kPadding1 = ByteStr("\0\0\0\0\0");
constexpr auto kEntry1 = AsBytes(kHeader1, kKey1, kValue1, kPadding1);
static_assert(kEntry1.size() == 32);

ChecksumCrc16 checksum;
constexpr EntryFormat kFormatWithChecksum{kMagicWithChecksum, &checksum};
constexpr internal::EntryFormats kFormats(kFormatWithChecksum);

class ValidEntryInFlash : public ::testing::Test {
protected:
ValidEntryInFlash() : flash_(kEntry1), partition_(&flash_) {
EXPECT_EQ(Status::OK, Entry::Read(partition_, 0, &entry_));
EXPECT_EQ(Status::OK, Entry::Read(partition_, 0, kFormats, &entry_));
}

FakeFlashBuffer<1024, 4> flash_;
Expand All @@ -105,13 +112,12 @@ class ValidEntryInFlash : public ::testing::Test {
};

TEST_F(ValidEntryInFlash, PassesChecksumVerification) {
ChecksumCrc16 checksum;
EXPECT_EQ(Status::OK, entry_.VerifyChecksumInFlash(&checksum));
EXPECT_EQ(Status::OK, entry_.VerifyChecksum(&checksum, "key45", kValue1));
EXPECT_EQ(Status::OK, entry_.VerifyChecksumInFlash());
EXPECT_EQ(Status::OK, entry_.VerifyChecksum("key45", kValue1));
}

TEST_F(ValidEntryInFlash, HeaderContents) {
EXPECT_EQ(entry_.magic(), 0x600DF00Du);
EXPECT_EQ(entry_.magic(), kMagicWithChecksum);
EXPECT_EQ(entry_.key_length(), 5u);
EXPECT_EQ(entry_.value_size(), 6u);
EXPECT_EQ(entry_.transaction_id(), 0x96979899u);
Expand Down Expand Up @@ -187,11 +193,9 @@ TEST_F(ValidEntryInFlash, ReadValue_WithOffset_PastEnd) {
TEST(ValidEntry, Write) {
FakeFlashBuffer<1024, 4> flash;
FlashPartition partition(&flash, 0, flash.sector_count(), 32);
ChecksumCrc16 checksum;
const EntryFormat format{0x600DF00Du, &checksum};

Entry entry =
Entry::Valid(partition, 53, format, "key45", kValue1, 0x96979899u);
Entry entry = Entry::Valid(
partition, 53, kFormatWithChecksum, "key45", kValue1, 0x96979899u);

auto result = entry.Write("key45", kValue1);
EXPECT_EQ(Status::OK, result.status());
Expand All @@ -215,7 +219,7 @@ class TombstoneEntryInFlash : public ::testing::Test {
protected:
TombstoneEntryInFlash()
: flash_(AsBytes(kHeader2, kKeyAndPadding2)), partition_(&flash_) {
EXPECT_EQ(Status::OK, Entry::Read(partition_, 0, &entry_));
EXPECT_EQ(Status::OK, Entry::Read(partition_, 0, kFormats, &entry_));
}

FakeFlashBuffer<1024, 4> flash_;
Expand All @@ -224,13 +228,12 @@ class TombstoneEntryInFlash : public ::testing::Test {
};

TEST_F(TombstoneEntryInFlash, PassesChecksumVerification) {
ChecksumCrc16 checksum;
EXPECT_EQ(Status::OK, entry_.VerifyChecksumInFlash(&checksum));
EXPECT_EQ(Status::OK, entry_.VerifyChecksum(&checksum, "K", {}));
EXPECT_EQ(Status::OK, entry_.VerifyChecksumInFlash());
EXPECT_EQ(Status::OK, entry_.VerifyChecksum("K", {}));
}

TEST_F(TombstoneEntryInFlash, HeaderContents) {
EXPECT_EQ(entry_.magic(), 0x600DF00Du);
EXPECT_EQ(entry_.magic(), kMagicWithChecksum);
EXPECT_EQ(entry_.key_length(), 1u);
EXPECT_EQ(entry_.value_size(), 0u);
EXPECT_EQ(entry_.transaction_id(), 0x03020100u);
Expand Down Expand Up @@ -258,9 +261,9 @@ TEST(TombstoneEntry, Write) {
FakeFlashBuffer<1024, 4> flash;
FlashPartition partition(&flash);
ChecksumCrc16 checksum;
const EntryFormat format{0x600DF00Du, &checksum};

Entry entry = Entry::Tombstone(partition, 16, format, "K", 0x03020100);
Entry entry =
Entry::Tombstone(partition, 16, kFormatWithChecksum, "K", 0x03020100);

auto result = entry.Write("K", {});
EXPECT_EQ(Status::OK, result.status());
Expand All @@ -275,33 +278,36 @@ TEST(Entry, Checksum_NoChecksumRequiresZero) {
FakeFlashBuffer<1024, 4> flash(kEntry1);
FlashPartition partition(&flash);
Entry entry;
ASSERT_EQ(Status::OK, Entry::Read(partition, 0, &entry));

EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksumInFlash(nullptr));
EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksum(nullptr, {}, {}));
const EntryFormat format{kMagicWithChecksum, nullptr};
const internal::EntryFormats formats(format);

ASSERT_EQ(Status::OK, Entry::Read(partition, 0, formats, &entry));

EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksumInFlash());
EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksum({}, {}));

std::memset(&flash.buffer()[4], 0, 4); // set the checksum field to 0
ASSERT_EQ(Status::OK, Entry::Read(partition, 0, &entry));
EXPECT_EQ(Status::OK, entry.VerifyChecksumInFlash(nullptr));
EXPECT_EQ(Status::OK, entry.VerifyChecksum(nullptr, {}, {}));
ASSERT_EQ(Status::OK, Entry::Read(partition, 0, formats, &entry));
EXPECT_EQ(Status::OK, entry.VerifyChecksumInFlash());
EXPECT_EQ(Status::OK, entry.VerifyChecksum({}, {}));
}

TEST(Entry, Checksum_ChecksPadding) {
FakeFlashBuffer<1024, 4> flash(
AsBytes(kHeader1, kKey1, kValue1, ByteStr("\0\0\0\0\1")));
FlashPartition partition(&flash);
Entry entry;
ASSERT_EQ(Status::OK, Entry::Read(partition, 0, &entry));
ASSERT_EQ(Status::OK, Entry::Read(partition, 0, kFormats, &entry));

// Last byte in padding is a 1; should fail.
ChecksumCrc16 checksum;
EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksumInFlash(&checksum));
EXPECT_EQ(Status::DATA_LOSS, entry.VerifyChecksumInFlash());

// The in-memory verification fills in 0s for the padding.
EXPECT_EQ(Status::OK, entry.VerifyChecksum(&checksum, "key45", kValue1));
EXPECT_EQ(Status::OK, entry.VerifyChecksum("key45", kValue1));

flash.buffer()[kEntry1.size() - 1] = byte{0};
EXPECT_EQ(Status::OK, entry.VerifyChecksumInFlash(&checksum));
EXPECT_EQ(Status::OK, entry.VerifyChecksumInFlash());
}
} // namespace
} // namespace pw::kvs::internal
28 changes: 28 additions & 0 deletions pw_kvs/format.cc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2020 The Pigweed Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

#include "pw_kvs/format.h"

namespace pw::kvs::internal {

const EntryFormat* EntryFormats::Find(const uint32_t magic) const {
for (const EntryFormat& format : formats_) {
if (format.magic == magic) {
return &format;
}
}
return nullptr;
}

} // namespace pw::kvs::internal
Loading

0 comments on commit 22d0d9f

Please sign in to comment.