From 2660c66c3105f504a8daf9ab37d752c9ccfb1529 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Tue, 13 Jun 2023 19:06:59 +0800 Subject: [PATCH] Pad the file for windows. --- rabit/include/rabit/internal/io.h | 14 +++++++++--- src/common/io.cc | 37 ++++++++++++++++++++++--------- src/common/io.h | 4 ++-- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/rabit/include/rabit/internal/io.h b/rabit/include/rabit/internal/io.h index 64633b2155aa..6a7d1227f4ae 100644 --- a/rabit/include/rabit/internal/io.h +++ b/rabit/include/rabit/internal/io.h @@ -6,6 +6,11 @@ */ #ifndef RABIT_INTERNAL_IO_H_ #define RABIT_INTERNAL_IO_H_ + +#if !defined(NOMINMAX) && defined(_WIN32) +#define NOMINMAX +#endif // !defined(NOMINMAX) + #include #include #include @@ -26,6 +31,9 @@ struct MemoryFixSizeBuffer : public SeekStream { // similar to SEEK_END in libc static size_t constexpr kSeekEnd = std::numeric_limits::max(); +protected: + MemoryFixSizeBuffer() = default; + public: MemoryFixSizeBuffer(void *p_buffer, size_t buffer_size) : p_buffer_(reinterpret_cast(p_buffer)), @@ -62,11 +70,11 @@ struct MemoryFixSizeBuffer : public SeekStream { protected: /*! \brief in memory buffer */ - char *p_buffer_; + char* p_buffer_{nullptr}; /*! \brief current pointer */ - size_t buffer_size_; + std::size_t buffer_size_{ 0 }; /*! \brief current pointer */ - size_t curr_ptr_; + std::size_t curr_ptr_{ 0 }; }; // class MemoryFixSizeBuffer /*! \brief a in memory buffer that can be read and write as stream interface */ diff --git a/src/common/io.cc b/src/common/io.cc index f6bf15fc4a3f..bf7c47ea2481 100644 --- a/src/common/io.cc +++ b/src/common/io.cc @@ -11,6 +11,7 @@ #include #include // for close, getpagesize #elif defined(_MSC_VER) +#define WIN32_LEAN_AND_MEAN #include #endif // defined(__unix__) @@ -26,6 +27,7 @@ #include "io.h" #include "xgboost/logging.h" +#include "xgboost/collective/socket.h" namespace xgboost { namespace common { @@ -170,7 +172,8 @@ std::size_t GetPageSize() { #if defined(_MSC_VER) SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); - return sys_info.dwPageSize; + // During testing, `sys_info.dwPageSize` is of size 4096 while `dwAllocationGranularity` is of size 65536. + return sys_info.dwAllocationGranularity; #else return getpagesize(); #endif @@ -190,18 +193,21 @@ std::size_t PadPageForMmap(std::size_t file_bytes, dmlc::Stream* fo) { struct PrivateMmapStream::MMAPFile { #if defined(_MSC_VER) - HANDLE fd; + HANDLE fd{ INVALID_HANDLE_VALUE }; #else - std::int32_t fd; + std::int32_t fd {0}; #endif std::string path; }; PrivateMmapStream::PrivateMmapStream(std::string path, bool read_only, std::size_t offset, std::size_t length) - : MemoryFixSizeBuffer{Open(std::move(path), read_only, offset, length), length} {} + : MemoryFixSizeBuffer{} { + this->p_buffer_ = Open(std::move(path), read_only, offset, length); + this->buffer_size_ = length; +} -void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offset, +char* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offset, std::size_t length) { #if defined(_MSC_VER) HANDLE fd = CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, @@ -211,7 +217,8 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs auto fd = open(path.c_str(), O_RDONLY); CHECK_GE(fd, 0) << "Failed to open:" << path << ". " << strerror(errno); #endif - handle_.reset(new MMAPFile{fd, std::move(path)}); + handle_ = nullptr; + handle_.reset(new MMAPFile{fd, path}); void* ptr{nullptr}; #if defined(__linux__) || defined(__GLIBC__) @@ -228,8 +235,12 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs access = read_only ? FILE_MAP_READ : FILE_MAP_ALL_ACCESS; std::uint32_t loff = static_cast(offset); std::uint32_t hoff = offset >> 32; + CHECK(map_file) << "Failed to map: " << handle_->path << ". " << GetLastError();; ptr = MapViewOfFile(map_file, access, hoff, loff, length); - CHECK_NE(ptr, nullptr) << "Failed to map: " << handle_->path << ". " << GetLastError(); + if (ptr == nullptr) { + system::ThrowAtError("MapViewOfFile"); + } + CHECK_NE(ptr, nullptr) << "Failed to map: " << handle_->path << ". " << GetLastError() ; #else CHECK_LE(offset, std::numeric_limits::max()) << "File size has exceeded the limit on the current system."; @@ -240,18 +251,22 @@ void* PrivateMmapStream::Open(std::string path, bool read_only, std::size_t offs ptr = reinterpret_cast(mmap(nullptr, length, prot, MAP_PRIVATE, fd_, offset)); CHECK_NE(ptr, MAP_FAILED) << "Failed to map: " << handle_->path << ". " << strerror(errno); #endif // defined(__linux__) - return ptr; + return reinterpret_cast(ptr); } PrivateMmapStream::~PrivateMmapStream() { CHECK(handle_); #if defined(_MSC_VER) - CHECK(UnmapViewOfFile(p_buffer_)) "Faled to munmap." << handle_->path << ". " << GetLastError(); - CloseHandle(handle_->fd); + if (p_buffer_) { + CHECK(UnmapViewOfFile(p_buffer_)) "Faled to munmap." << GetLastError(); + } + if (handle_->fd != INVALID_HANDLE_VALUE) { + CHECK(CloseHandle(handle_->fd)); + } #else CHECK_NE(munmap(p_buffer_, buffer_size_), -1) << "Faled to munmap." << handle_->path << ". " << strerror(errno); - CHECK_NE(close(fd_), -1) << "Faled to close: " << handle_->path << ". " << strerror(errno); + CHECK_NE(close(handle_->fd), -1) << "Faled to close: " << handle_->path << ". " << strerror(errno); #endif } } // namespace common diff --git a/src/common/io.h b/src/common/io.h index cd82782b025b..bdebc5ac3298 100644 --- a/src/common/io.h +++ b/src/common/io.h @@ -146,9 +146,9 @@ std::size_t PadPageForMmap(std::size_t file_bytes, dmlc::Stream* fo); class PrivateMmapStream : public MemoryFixSizeBuffer { struct MMAPFile; - std::unique_ptr handle_; + std::unique_ptr handle_{nullptr}; - void* Open(std::string path, bool read_only, std::size_t offset, std::size_t length); + char* Open(std::string path, bool read_only, std::size_t offset, std::size_t length); public: /**