Skip to content

Commit

Permalink
Introduce BaseFileReader::get_mmap_buffer.
Browse files Browse the repository at this point in the history
BaseFileReader is now responsible for trying to get a mmap and then
fallback to reading in file.
  • Loading branch information
mgautierfr committed Apr 3, 2024
1 parent a77ab8b commit e333968
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 37 deletions.
62 changes: 29 additions & 33 deletions src/file_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -177,24 +177,11 @@ makeMmappedBuffer(int fd, offset_t offset, zsize_t size)
} // unnamed namespace
#endif // ENABLE_USE_MMAP

const Buffer MultiPartFileReader::get_buffer(offset_t offset, zsize_t size) const {
const Buffer BaseFileReader::get_buffer(offset_t offset, zsize_t size) const {
ASSERT(size, <=, _size);
#ifdef ENABLE_USE_MMAP
try {
auto found_range = source->locate(_offset+offset, size);
auto first_part_containing_it = found_range.first;
if (++first_part_containing_it != found_range.second) {
throw MMapException();
}

// The range is in only one part
auto range = found_range.first->first;
auto part = found_range.first->second;
auto logical_local_offset = offset + _offset - range.min;
ASSERT(size, <=, part->size());
int fd = part->fhandle().getNativeHandle();
auto physical_local_offset = logical_local_offset + part->offset();
return Buffer::makeBuffer(makeMmappedBuffer(fd, physical_local_offset, size), size);
return get_mmap_buffer(offset, size);
} catch(MMapException& e)
#endif
{
Expand All @@ -210,6 +197,27 @@ const Buffer MultiPartFileReader::get_buffer(offset_t offset, zsize_t size) cons
}
}

const Buffer MultiPartFileReader::get_mmap_buffer(offset_t offset, zsize_t size) const {
#ifdef ENABLE_USE_MMAP
auto found_range = source->locate(_offset + offset, size);
auto first_part_containing_it = found_range.first;
if (++first_part_containing_it != found_range.second) {
throw MMapException();
}

// The range is in only one part
auto range = found_range.first->first;
auto part = found_range.first->second;
auto logical_local_offset = offset + _offset - range.min;
ASSERT(size, <=, part->size());
int fd = part->fhandle().getNativeHandle();
auto physical_local_offset = logical_local_offset + part->offset();
return Buffer::makeBuffer(makeMmappedBuffer(fd, physical_local_offset, size), size);
#else
return Buffer::makeBuffer(size); // unreachable
#endif
}

bool Reader::can_read(offset_t offset, zsize_t size) const
{
return (offset.v <= this->size().v && (offset.v+size.v) <= this->size().v);
Expand Down Expand Up @@ -273,26 +281,14 @@ void FileReader::read(char* dest, offset_t offset, zsize_t size) const
};
}

const Buffer FileReader::get_buffer(offset_t offset, zsize_t size) const
{
ASSERT(size, <=, _size);
const Buffer FileReader::get_mmap_buffer(offset_t offset, zsize_t size) const {
#ifdef ENABLE_USE_MMAP
try {
auto local_offset = offset + _offset;
int fd = _fhandle->getNativeHandle();
return Buffer::makeBuffer(makeMmappedBuffer(fd, local_offset, size), size);
} catch(MMapException& e)
auto local_offset = offset + _offset;
int fd = _fhandle->getNativeHandle();
return Buffer::makeBuffer(makeMmappedBuffer(fd, local_offset, size), size);
#else
return Buffer::makeBuffer(size); // unreachable
#endif
{
// We cannot do the mmap, for several possible reasons:
// - Mmap offset is too big (>4GB on 32 bits)
// - We are on Windows.
// We will have to do some memory copies :/
// [TODO] Use Windows equivalent for mmap.
auto ret_buffer = Buffer::makeBuffer(size);
read(const_cast<char*>(ret_buffer.data()), offset, size);
return ret_buffer;
}
}

std::unique_ptr<const Reader>
Expand Down
12 changes: 8 additions & 4 deletions src/file_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ class BaseFileReader : public Reader {
zsize_t size() const { return _size; };
offset_t offset() const { return _offset; };

virtual const Buffer get_mmap_buffer(offset_t offset,
zsize_t size) const = 0;
const Buffer get_buffer(offset_t offset, zsize_t size) const;

protected: // data
offset_t _offset;
zsize_t _size;
Expand All @@ -50,9 +54,9 @@ class FileReader : public BaseFileReader {
~FileReader() = default;

char read(offset_t offset) const;
void read(char* dest, offset_t offset, zsize_t size) const;
const Buffer get_buffer(offset_t offset, zsize_t size) const;
void read(char *dest, offset_t offset, zsize_t size) const;

const Buffer get_mmap_buffer(offset_t offset, zsize_t size) const;
std::unique_ptr<const Reader> sub_reader(offset_t offset, zsize_t size) const;

private: // data
Expand All @@ -68,9 +72,9 @@ class MultiPartFileReader : public BaseFileReader {
~MultiPartFileReader() {};

char read(offset_t offset) const;
void read(char* dest, offset_t offset, zsize_t size) const;
const Buffer get_buffer(offset_t offset, zsize_t size) const;
void read(char *dest, offset_t offset, zsize_t size) const;

const Buffer get_mmap_buffer(offset_t offset, zsize_t size) const;
std::unique_ptr<const Reader> sub_reader(offset_t offset, zsize_t size) const;

private:
Expand Down

0 comments on commit e333968

Please sign in to comment.