From 2dd96d5abe61fa7fd2a834aa60345c2fd5e4c1a2 Mon Sep 17 00:00:00 2001 From: Igor Novikov Date: Sat, 5 Oct 2024 01:18:13 +0500 Subject: [PATCH 1/2] Added IBufferedFileStream - a faster variant of IFileStream designed for working with small files such as kosaves. --- cmake/headerlist.cmake | 1 + cmake/sourcelist.cmake | 1 + common/IBufferedFileStream.cpp | 120 +++++++++++++++++++++++++++++++++ common/IBufferedFileStream.h | 33 +++++++++ common/IFileStream.h | 16 ++--- 5 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 common/IBufferedFileStream.cpp create mode 100644 common/IBufferedFileStream.h diff --git a/cmake/headerlist.cmake b/cmake/headerlist.cmake index 0bc9ea9..bf21c40 100644 --- a/cmake/headerlist.cmake +++ b/cmake/headerlist.cmake @@ -1,5 +1,6 @@ set(headers common/IArchive.h + common/IBufferedFileStream.h common/IBufferStream.h common/IConsole.h common/ICriticalSection.h diff --git a/cmake/sourcelist.cmake b/cmake/sourcelist.cmake index 558e814..f77e7be 100644 --- a/cmake/sourcelist.cmake +++ b/cmake/sourcelist.cmake @@ -1,5 +1,6 @@ set(sources common/IArchive.cpp + common/IBufferedFileStream.cpp common/IBufferStream.cpp common/IConsole.cpp common/IDataStream.cpp diff --git a/common/IBufferedFileStream.cpp b/common/IBufferedFileStream.cpp new file mode 100644 index 0000000..9739e67 --- /dev/null +++ b/common/IBufferedFileStream.cpp @@ -0,0 +1,120 @@ +#include "IBufferedFileStream.h" + +IBufferedFileStream::IBufferedFileStream() { } + +IBufferedFileStream::~IBufferedFileStream() +{ + Close(); + + free(streamBuffer); +} + +bool IBufferedFileStream::Open(const char* name) +{ + bool success = IFileStream::Open(name); + + if (success) + { + ReallocateStreamBuffer(streamLength); + + IFileStream::ReadBuf(streamBuffer, streamLength); + + IFileStream::SetOffset(0); + } + + return success; +} + +bool IBufferedFileStream::Create(const char* name) +{ + bool success = IFileStream::Create(name); + + if (success) + { + ReallocateStreamBuffer(streamLength); + } + + return success; +} + +void IBufferedFileStream::Close(void) +{ + if (streamBufferHasWrite) + { + IFileStream::SetOffset(0); + + IFileStream::WriteBuf(streamBuffer, streamLength); + } + + IFileStream::Close(); +} + +void IBufferedFileStream::ReadBuf(void* buf, UInt32 inLength) +{ + ASSERT_STR(inLength <= GetRemain(), "IBufferedFileStream::ReadBuf: hit eof"); + ASSERT_STR(streamBufferLength >= streamOffset + inLength, "IBufferedFileStream::ReadBuf: hit buffer eof"); + + memcpy(buf, &streamBuffer[streamOffset], inLength); + streamOffset += inLength; +} + +void IBufferedFileStream::WriteBuf(const void* buf, UInt32 inLength) +{ + streamBufferHasWrite = true; + + if (streamBufferLength < streamOffset + inLength) + ReallocateStreamBuffer(streamOffset + inLength); + + memcpy(&streamBuffer[streamOffset], buf, inLength); + streamOffset += inLength; + + if (streamLength < streamOffset) + streamLength = streamOffset; +} + +void IBufferedFileStream::SetOffset(SInt64 inOffset) +{ + IDataStream::SetOffset(inOffset); +} + +void IBufferedFileStream::SetLength(UInt64 length) +{ + ReallocateStreamBuffer(length); + + IFileStream::SetLength(length); +} + +void IBufferedFileStream::ReallocateStreamBuffer(UInt64 length) +{ + UInt64 newStreamBufferLength = CalculateStreamBufferLength(length); + + if (newStreamBufferLength != streamBufferLength) + { + streamBufferLength = newStreamBufferLength; + + if (streamBuffer == nullptr) + { + streamBuffer = (UInt8*)malloc(streamBufferLength); + } + else + { + UInt8* newStreamBuffer = (UInt8*)realloc(streamBuffer, streamBufferLength); + + ASSERT_STR(newStreamBuffer, "IBufferedFileStream::ReallocateBuffer: realloc failed"); + + streamBuffer = newStreamBuffer; + } + } +} + +UInt64 IBufferedFileStream::CalculateStreamBufferLength(UInt64 length) +{ + const UInt64 step = 1024ull * 1024ull; + + if (!length) return step; + + UInt64 whole = length / step; + UInt64 remain = length % step; + + return (whole + (remain > 0 ? 1 : 0)) * step; +} diff --git a/common/IBufferedFileStream.h b/common/IBufferedFileStream.h new file mode 100644 index 0000000..5175f41 --- /dev/null +++ b/common/IBufferedFileStream.h @@ -0,0 +1,33 @@ +#pragma once + +#include "common/IDataStream.h" +#include "common/IFileStream.h" + +/** + * An input/output file stream with buffering, write file on close + */ +class IBufferedFileStream : public IFileStream +{ +public: + IBufferedFileStream(); + virtual ~IBufferedFileStream(); + + virtual bool Open(const char* name); + virtual bool Create(const char* name); + + virtual void Close(void); + + virtual void ReadBuf(void* buf, UInt32 inLength); + virtual void WriteBuf(const void* buf, UInt32 inLength); + virtual void SetOffset(SInt64 inOffset); + + virtual void SetLength(UInt64 length); + +private: + void ReallocateStreamBuffer(UInt64 length); + UInt64 CalculateStreamBufferLength(UInt64 length); + + UInt8* streamBuffer = nullptr; + UInt64 streamBufferLength = 0; + bool streamBufferHasWrite = false; +}; diff --git a/common/IFileStream.h b/common/IFileStream.h index 01aa955..a0a774b 100644 --- a/common/IFileStream.h +++ b/common/IFileStream.h @@ -3,7 +3,7 @@ #include "common/IDataStream.h" /** - * An input file stream + * An input/output file stream */ class IFileStream : public IDataStream { @@ -12,22 +12,22 @@ class IFileStream : public IDataStream IFileStream(const char * name); ~IFileStream(); - bool Open(const char * name); - bool BrowseOpen(void); + virtual bool Open(const char * name); + bool BrowseOpen(void); - bool Create(const char * name); - bool BrowseCreate(const char * defaultName = NULL, const char * defaultPath = NULL, const char * title = NULL); + virtual bool Create(const char * name); + bool BrowseCreate(const char * defaultName = NULL, const char * defaultPath = NULL, const char * title = NULL); - void Close(void); + virtual void Close(void); - HANDLE GetHandle(void) { return theFile; } + HANDLE GetHandle(void) { return theFile; } virtual void ReadBuf(void * buf, UInt32 inLength); virtual void WriteBuf(const void * buf, UInt32 inLength); virtual void SetOffset(SInt64 inOffset); // can truncate. implicitly seeks to the end of the file - void SetLength(UInt64 length); + virtual void SetLength(UInt64 length); static void MakeAllDirs(const char * path); static char * ExtractFileName(char * path); From d7ef5e2e96afac0e36545d40713669ee86d26302 Mon Sep 17 00:00:00 2001 From: Igor Novikov Date: Wed, 4 Dec 2024 22:57:45 +0500 Subject: [PATCH 2/2] Rewrote IBufferedFileStream - now using std::vector and a separate IFileStream for reading/writing. --- common/IBufferedFileStream.cpp | 82 ++++++++++------------------------ common/IBufferedFileStream.h | 18 ++++---- common/IFileStream.h | 14 +++--- 3 files changed, 38 insertions(+), 76 deletions(-) diff --git a/common/IBufferedFileStream.cpp b/common/IBufferedFileStream.cpp index 9739e67..ef5f60b 100644 --- a/common/IBufferedFileStream.cpp +++ b/common/IBufferedFileStream.cpp @@ -5,21 +5,20 @@ IBufferedFileStream::IBufferedFileStream() { } IBufferedFileStream::~IBufferedFileStream() { Close(); - - free(streamBuffer); } bool IBufferedFileStream::Open(const char* name) { - bool success = IFileStream::Open(name); + bool success = streamFile.Open(name); if (success) { - ReallocateStreamBuffer(streamLength); + SetLength(streamFile.GetLength()); + SetOffset(0); - IFileStream::ReadBuf(streamBuffer, streamLength); + streamFile.ReadBuf(streamBuffer.data(), streamLength); - IFileStream::SetOffset(0); + streamFile.SetOffset(0); } return success; @@ -27,11 +26,12 @@ bool IBufferedFileStream::Open(const char* name) bool IBufferedFileStream::Create(const char* name) { - bool success = IFileStream::Create(name); + bool success = streamFile.Create(name); if (success) { - ReallocateStreamBuffer(streamLength); + SetLength(0); + SetOffset(0); } return success; @@ -41,18 +41,25 @@ void IBufferedFileStream::Close(void) { if (streamBufferHasWrite) { - IFileStream::SetOffset(0); + streamFile.SetLength(streamLength); + + streamFile.SetOffset(0); + + streamFile.WriteBuf(streamBuffer.data(), streamLength); - IFileStream::WriteBuf(streamBuffer, streamLength); + streamBufferHasWrite = false; } - IFileStream::Close(); + SetLength(0); + SetOffset(0); + + streamFile.Close(); } void IBufferedFileStream::ReadBuf(void* buf, UInt32 inLength) { ASSERT_STR(inLength <= GetRemain(), "IBufferedFileStream::ReadBuf: hit eof"); - ASSERT_STR(streamBufferLength >= streamOffset + inLength, "IBufferedFileStream::ReadBuf: hit buffer eof"); + ASSERT_STR(streamBuffer.size() >= streamOffset + inLength, "IBufferedFileStream::ReadBuf: hit buffer eof"); memcpy(buf, &streamBuffer[streamOffset], inLength); streamOffset += inLength; @@ -62,59 +69,16 @@ void IBufferedFileStream::WriteBuf(const void* buf, UInt32 inLength) { streamBufferHasWrite = true; - if (streamBufferLength < streamOffset + inLength) - ReallocateStreamBuffer(streamOffset + inLength); + if (streamBuffer.size() < streamOffset + inLength) + SetLength(streamOffset + inLength); memcpy(&streamBuffer[streamOffset], buf, inLength); streamOffset += inLength; - - if (streamLength < streamOffset) - streamLength = streamOffset; -} - -void IBufferedFileStream::SetOffset(SInt64 inOffset) -{ - IDataStream::SetOffset(inOffset); } void IBufferedFileStream::SetLength(UInt64 length) { - ReallocateStreamBuffer(length); - - IFileStream::SetLength(length); -} - -void IBufferedFileStream::ReallocateStreamBuffer(UInt64 length) -{ - UInt64 newStreamBufferLength = CalculateStreamBufferLength(length); - - if (newStreamBufferLength != streamBufferLength) - { - streamBufferLength = newStreamBufferLength; - - if (streamBuffer == nullptr) - { - streamBuffer = (UInt8*)malloc(streamBufferLength); - } - else - { - UInt8* newStreamBuffer = (UInt8*)realloc(streamBuffer, streamBufferLength); - - ASSERT_STR(newStreamBuffer, "IBufferedFileStream::ReallocateBuffer: realloc failed"); - - streamBuffer = newStreamBuffer; - } - } -} - -UInt64 IBufferedFileStream::CalculateStreamBufferLength(UInt64 length) -{ - const UInt64 step = 1024ull * 1024ull; - - if (!length) return step; - - UInt64 whole = length / step; - UInt64 remain = length % step; + streamBuffer.resize(length); - return (whole + (remain > 0 ? 1 : 0)) * step; + streamLength = length; } diff --git a/common/IBufferedFileStream.h b/common/IBufferedFileStream.h index 5175f41..6e453d3 100644 --- a/common/IBufferedFileStream.h +++ b/common/IBufferedFileStream.h @@ -2,32 +2,30 @@ #include "common/IDataStream.h" #include "common/IFileStream.h" +#include /** * An input/output file stream with buffering, write file on close */ -class IBufferedFileStream : public IFileStream +class IBufferedFileStream : public IDataStream { public: IBufferedFileStream(); virtual ~IBufferedFileStream(); - virtual bool Open(const char* name); - virtual bool Create(const char* name); + bool Open(const char* name); + bool Create(const char* name); - virtual void Close(void); + void Close(void); virtual void ReadBuf(void* buf, UInt32 inLength); virtual void WriteBuf(const void* buf, UInt32 inLength); - virtual void SetOffset(SInt64 inOffset); - virtual void SetLength(UInt64 length); + void SetLength(UInt64 length); private: - void ReallocateStreamBuffer(UInt64 length); - UInt64 CalculateStreamBufferLength(UInt64 length); + std::vector streamBuffer; + IFileStream streamFile; - UInt8* streamBuffer = nullptr; - UInt64 streamBufferLength = 0; bool streamBufferHasWrite = false; }; diff --git a/common/IFileStream.h b/common/IFileStream.h index a0a774b..2e0208a 100644 --- a/common/IFileStream.h +++ b/common/IFileStream.h @@ -12,22 +12,22 @@ class IFileStream : public IDataStream IFileStream(const char * name); ~IFileStream(); - virtual bool Open(const char * name); - bool BrowseOpen(void); + bool Open(const char * name); + bool BrowseOpen(void); - virtual bool Create(const char * name); - bool BrowseCreate(const char * defaultName = NULL, const char * defaultPath = NULL, const char * title = NULL); + bool Create(const char * name); + bool BrowseCreate(const char * defaultName = NULL, const char * defaultPath = NULL, const char * title = NULL); - virtual void Close(void); + void Close(void); - HANDLE GetHandle(void) { return theFile; } + HANDLE GetHandle(void) { return theFile; } virtual void ReadBuf(void * buf, UInt32 inLength); virtual void WriteBuf(const void * buf, UInt32 inLength); virtual void SetOffset(SInt64 inOffset); // can truncate. implicitly seeks to the end of the file - virtual void SetLength(UInt64 length); + void SetLength(UInt64 length); static void MakeAllDirs(const char * path); static char * ExtractFileName(char * path);