Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Skinny] Add AppendWithVerify and PositionedAppendWithVerify to Env and FileSystem #7419

Closed
wants to merge 3 commits into from
Closed

[Skinny] Add AppendWithVerify and PositionedAppendWithVerify to Env and FileSystem #7419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b57cbeb
Add AppendWithVerify and PositionedAppendWithVerify to Env and FileSy…
zhichao-cao Sep 21, 2020
c800474
Address the comments
zhichao-cao Sep 22, 2020
0dc5f5f
move the DataVerificationInfo before debug
zhichao-cao Sep 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions env/composite_env_wrapper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,8 +727,19 @@ class LegacyWritableFileWrapper : public FSWritableFile {
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->Append(data));
}
IOStatus Append(const Slice& data, const IOOptions& /*options*/,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->Append(data));
}
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->PositionedAppend(data, offset));
}
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& /*options*/,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->PositionedAppend(data, offset));
}
Expand Down
11 changes: 11 additions & 0 deletions env/file_system_tracer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,10 +244,21 @@ class FSWritableFileTracingWrapper : public FSWritableFileWrapper {

IOStatus Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Append(const Slice& data, const IOOptions& options,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* dbg) override {
return Append(data, options, dbg);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be implemented in file_system_tracer.cc rather than redirecting here.

}

IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
IODebugContext* dbg) override;
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* dbg) override {
return PositionedAppend(data, offset, options, dbg);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment as above

}

IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override;
Expand Down
16 changes: 16 additions & 0 deletions env/io_posix.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,9 +242,20 @@ class PosixWritableFile : public FSWritableFile {
virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Append(const Slice& data, const IOOptions& opts,
IODebugContext* dbg) override;
virtual IOStatus Append(const Slice& data, const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return Append(data, opts, dbg);
}
virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& opts,
IODebugContext* dbg) override;
virtual IOStatus PositionedAppend(
const Slice& data, uint64_t offset, const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return PositionedAppend(data, offset, opts, dbg);
}
virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;
Expand Down Expand Up @@ -331,6 +342,11 @@ class PosixMmapFile : public FSWritableFile {
virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Append(const Slice& data, const IOOptions& opts,
IODebugContext* dbg) override;
virtual IOStatus Append(const Slice& data, const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return Append(data, opts, dbg);
}
virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override;
virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;
Expand Down
41 changes: 41 additions & 0 deletions include/rocksdb/file_system.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,13 @@ class FSRandomAccessFile {
// RandomAccessFileWrapper too.
};

// A data structure brings the data verification information, which is
// used togther with data being written to a file.
struct DataVerificationInfo {
// checksum of the data being written.
Slice checksum;
};

// A file abstraction for sequential writing. The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
Expand Down Expand Up @@ -729,6 +736,16 @@ class FSWritableFile {
virtual IOStatus Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) = 0;

// EXPERIMENTAL / CURRENTLY UNUSED
// Append data with verification information
// Note that this API change is experimental and it might be changed in
// the future. Currently, RocksDB does not use this API.
virtual IOStatus Append(const Slice& data, const IOOptions& options,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) {
return Append(data, options, dbg);
}

// PositionedAppend data to the specified offset. The new EOF after append
// must be larger than the previous EOF. This is to be used when writes are
// not backed by OS buffers and hence has to always start from the start of
Expand Down Expand Up @@ -756,6 +773,18 @@ class FSWritableFile {
return IOStatus::NotSupported();
}

// EXPERIMENTAL / CURRENTLY UNUSED
// PositionedAppend data with verification information.
// Note that this API change is experimental and it might be changed in
// the future. Currently, RocksDB does not use this API.
virtual IOStatus PositionedAppend(
const Slice& /* data */, uint64_t /* offset */,
const IOOptions& /*options*/,
const DataVerificationInfo& /* verification_info */,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
}

// Truncate is necessary to trim the file to the correct size
// before closing. It is not always possible to keep track of the file
// size due to whole pages writes. The behavior is undefined if called
Expand Down Expand Up @@ -1286,11 +1315,23 @@ class FSWritableFileWrapper : public FSWritableFile {
IODebugContext* dbg) override {
return target_->Append(data, options, dbg);
}
IOStatus Append(const Slice& data, const IOOptions& options,
const DataVerificationInfo& verification_info,
IODebugContext* dbg) override {
return target_->Append(data, options, verification_info, dbg);
}
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
IODebugContext* dbg) override {
return target_->PositionedAppend(data, offset, options, dbg);
}
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
const DataVerificationInfo& verification_info,
IODebugContext* dbg) override {
return target_->PositionedAppend(data, offset, options, verification_info,
dbg);
}
IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override {
return target_->Truncate(size, options, dbg);
Expand Down
11 changes: 11 additions & 0 deletions utilities/fault_injection_fs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ class TestFSWritableFile : public FSWritableFile {
virtual ~TestFSWritableFile();
virtual IOStatus Append(const Slice& data, const IOOptions&,
IODebugContext*) override;
virtual IOStatus Append(const Slice& data, const IOOptions& options,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* dbg) override {
return Append(data, options, dbg);
}
virtual IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override {
return target_->Truncate(size, options, dbg);
Expand All @@ -78,6 +83,12 @@ class TestFSWritableFile : public FSWritableFile {
IODebugContext* dbg) override {
return target_->PositionedAppend(data, offset, options, dbg);
}
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
const DataVerificationInfo& /*verification_info*/,
IODebugContext* dbg) override {
return PositionedAppend(data, offset, options, dbg);
}
virtual size_t GetRequiredBufferAlignment() const override {
return target_->GetRequiredBufferAlignment();
}
Expand Down