Skip to content

Commit

Permalink
Make all file access 64-bit (uint64_t)
Browse files Browse the repository at this point in the history
This changes the types of a big number of variables.

General rules:
- Using `uint64_t` in general. We also considered `int64_t` but eventually
  settled on keeping it unsigned, which is also closer to what one would expect
  with `size_t`/`off_t`.
- We only keep `int64_t` for `seek_end` (takes a negative offset from the end)
  and for the `Variant` bindings, since `Variant::INT` is `int64_t`. This means
  we only need to guard against passing negative values in `core_bind.cpp`.
- Using `uint32_t` integers for concepts not needing such a huge range, like
  pages, blocks, etc.

In addition:
- Improve usage of integer types in some related places; namely, `DirAccess`,
  core binds.

Note:
- On Windows, `_ftelli64` reports invalid values when using 32-bit MinGW with
  version < 8.0. This was an upstream bug fixed in 8.0. It breaks support for
  big files on 32-bit Windows builds made with that toolchain. We might add a
  workaround.

Fixes #44363.
Fixes godotengine/godot-proposals#400.

Co-authored-by: Rémi Verschelde <rverschelde@gmail.com>
  • Loading branch information
RandomShaper and akien-mga committed May 16, 2021
1 parent 7417467 commit 817ffc0
Show file tree
Hide file tree
Showing 66 changed files with 353 additions and 364 deletions.
19 changes: 11 additions & 8 deletions core/bind/core_bind.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1900,18 +1900,21 @@ String _File::get_path_absolute() const {

void _File::seek(int64_t p_position) {
ERR_FAIL_COND_MSG(!f, "File must be opened before use.");
ERR_FAIL_COND_MSG(p_position < 0, "Seek position must be a positive integer.");
f->seek(p_position);
}

void _File::seek_end(int64_t p_position) {
ERR_FAIL_COND_MSG(!f, "File must be opened before use.");
f->seek_end(p_position);
}
int64_t _File::get_position() const {

uint64_t _File::get_position() const {
ERR_FAIL_COND_V_MSG(!f, 0, "File must be opened before use.");
return f->get_position();
}

int64_t _File::get_len() const {
uint64_t _File::get_len() const {
ERR_FAIL_COND_V_MSG(!f, 0, "File must be opened before use.");
return f->get_len();
}
Expand Down Expand Up @@ -1951,7 +1954,7 @@ real_t _File::get_real() const {
return f->get_real();
}

PoolVector<uint8_t> _File::get_buffer(int p_length) const {
PoolVector<uint8_t> _File::get_buffer(int64_t p_length) const {
PoolVector<uint8_t> data;
ERR_FAIL_COND_V_MSG(!f, data, "File must be opened before use.");

Expand All @@ -1964,7 +1967,7 @@ PoolVector<uint8_t> _File::get_buffer(int p_length) const {
ERR_FAIL_COND_V_MSG(err != OK, data, "Can't resize data to " + itos(p_length) + " elements.");

PoolVector<uint8_t>::Write w = data.write();
int len = f->get_buffer(&w[0], p_length);
int64_t len = f->get_buffer(&w[0], p_length);
ERR_FAIL_COND_V(len < 0, PoolVector<uint8_t>());

w.release();
Expand All @@ -1980,7 +1983,7 @@ String _File::get_as_text() const {
ERR_FAIL_COND_V_MSG(!f, String(), "File must be opened before use.");

String text;
size_t original_pos = f->get_position();
uint64_t original_pos = f->get_position();
f->seek(0);

String l = get_line();
Expand Down Expand Up @@ -2103,7 +2106,7 @@ void _File::store_csv_line(const Vector<String> &p_values, const String &p_delim
void _File::store_buffer(const PoolVector<uint8_t> &p_buffer) {
ERR_FAIL_COND_MSG(!f, "File must be opened before use.");

int len = p_buffer.size();
uint64_t len = p_buffer.size();
if (len == 0) {
return;
}
Expand Down Expand Up @@ -2341,9 +2344,9 @@ bool _Directory::dir_exists(String p_dir) {
}
}

int _Directory::get_space_left() {
uint64_t _Directory::get_space_left() {
ERR_FAIL_COND_V_MSG(!d, 0, "Directory must be opened before use.");
return d->get_space_left() / 1024 * 1024; //return value in megabytes, given binding is int
return d->get_space_left() / 1024 * 1024; // Truncate to closest MiB.
}

Error _Directory::copy(String p_from, String p_to) {
Expand Down
8 changes: 4 additions & 4 deletions core/bind/core_bind.h
Original file line number Diff line number Diff line change
Expand Up @@ -516,8 +516,8 @@ class _File : public Reference {

void seek(int64_t p_position); // Seek to a given position.
void seek_end(int64_t p_position = 0); // Seek from the end of file.
int64_t get_position() const; // Get position in the file.
int64_t get_len() const; // Get size of the file.
uint64_t get_position() const; // Get position in the file.
uint64_t get_len() const; // Get size of the file.

bool eof_reached() const; // Reading passed EOF.

Expand All @@ -532,7 +532,7 @@ class _File : public Reference {

Variant get_var(bool p_allow_objects = false) const;

PoolVector<uint8_t> get_buffer(int p_length) const; // Get an array of bytes.
PoolVector<uint8_t> get_buffer(int64_t p_length) const; // Get an array of bytes.
String get_line() const;
Vector<String> get_csv_line(const String &p_delim = ",") const;
String get_as_text() const;
Expand Down Expand Up @@ -609,7 +609,7 @@ class _Directory : public Reference {
bool file_exists(String p_file);
bool dir_exists(String p_dir);

int get_space_left();
uint64_t get_space_left();

Error copy(String p_from, String p_to);
Error rename(String p_from, String p_to);
Expand Down
37 changes: 20 additions & 17 deletions core/io/file_access_compressed.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

#include "core/print_string.h"

void FileAccessCompressed::configure(const String &p_magic, Compression::Mode p_mode, int p_block_size) {
void FileAccessCompressed::configure(const String &p_magic, Compression::Mode p_mode, uint32_t p_block_size) {
magic = p_magic.ascii().get_data();
if (magic.length() > 4) {
magic = magic.substr(0, 4);
Expand Down Expand Up @@ -67,10 +67,10 @@ Error FileAccessCompressed::open_after_magic(FileAccess *p_base) {
ERR_FAIL_V_MSG(ERR_FILE_CORRUPT, "Can't open compressed file '" + p_base->get_path() + "' with block size 0, it is corrupted.");
}
read_total = f->get_32();
int bc = (read_total / block_size) + 1;
int acc_ofs = f->get_position() + bc * 4;
int max_bs = 0;
for (int i = 0; i < bc; i++) {
uint32_t bc = (read_total / block_size) + 1;
uint64_t acc_ofs = f->get_position() + bc * 4;
uint32_t max_bs = 0;
for (uint32_t i = 0; i < bc; i++) {
ReadBlock rb;
rb.offset = acc_ofs;
rb.csize = f->get_32();
Expand Down Expand Up @@ -147,15 +147,15 @@ void FileAccessCompressed::close() {
f->store_32(cmode); //write compression mode 4
f->store_32(block_size); //write block size 4
f->store_32(write_max); //max amount of data written 4
int bc = (write_max / block_size) + 1;
uint32_t bc = (write_max / block_size) + 1;

for (int i = 0; i < bc; i++) {
for (uint32_t i = 0; i < bc; i++) {
f->store_32(0); //compressed sizes, will update later
}

Vector<int> block_sizes;
for (int i = 0; i < bc; i++) {
int bl = i == (bc - 1) ? write_max % block_size : block_size;
for (uint32_t i = 0; i < bc; i++) {
uint32_t bl = i == (bc - 1) ? write_max % block_size : block_size;
uint8_t *bp = &write_ptr[i * block_size];

Vector<uint8_t> cblock;
Expand All @@ -167,7 +167,7 @@ void FileAccessCompressed::close() {
}

f->seek(16); //ok write block sizes
for (int i = 0; i < bc; i++) {
for (uint32_t i = 0; i < bc; i++) {
f->store_32(block_sizes[i]);
}
f->seek_end();
Expand All @@ -189,8 +189,9 @@ bool FileAccessCompressed::is_open() const {
return f != nullptr;
}

void FileAccessCompressed::seek(size_t p_position) {
void FileAccessCompressed::seek(uint64_t p_position) {
ERR_FAIL_COND_MSG(!f, "File must be opened before use.");

if (writing) {
ERR_FAIL_COND(p_position > write_max);

Expand All @@ -203,7 +204,7 @@ void FileAccessCompressed::seek(size_t p_position) {
} else {
at_end = false;
read_eof = false;
int block_idx = p_position / block_size;
uint32_t block_idx = p_position / block_size;
if (block_idx != read_block) {
read_block = block_idx;
f->seek(read_blocks[read_block].offset);
Expand All @@ -225,15 +226,17 @@ void FileAccessCompressed::seek_end(int64_t p_position) {
seek(read_total + p_position);
}
}
size_t FileAccessCompressed::get_position() const {

uint64_t FileAccessCompressed::get_position() const {
ERR_FAIL_COND_V_MSG(!f, 0, "File must be opened before use.");
if (writing) {
return write_pos;
} else {
return read_block * block_size + read_pos;
}
}
size_t FileAccessCompressed::get_len() const {

uint64_t FileAccessCompressed::get_len() const {
ERR_FAIL_COND_V_MSG(!f, 0, "File must be opened before use.");
if (writing) {
return write_max;
Expand Down Expand Up @@ -281,9 +284,9 @@ uint8_t FileAccessCompressed::get_8() const {

return ret;
}
int FileAccessCompressed::get_buffer(uint8_t *p_dst, int p_length) const {

uint64_t FileAccessCompressed::get_buffer(uint8_t *p_dst, uint64_t p_length) const {
ERR_FAIL_COND_V(!p_dst && p_length > 0, -1);
ERR_FAIL_COND_V(p_length < 0, -1);
ERR_FAIL_COND_V_MSG(!f, -1, "File must be opened before use.");
ERR_FAIL_COND_V_MSG(writing, -1, "File has not been opened in read mode.");

Expand All @@ -292,7 +295,7 @@ int FileAccessCompressed::get_buffer(uint8_t *p_dst, int p_length) const {
return 0;
}

for (int i = 0; i < p_length; i++) {
for (uint64_t i = 0; i < p_length; i++) {
p_dst[i] = read_ptr[read_pos];
read_pos++;
if (read_pos >= read_block_size) {
Expand Down
28 changes: 14 additions & 14 deletions core/io/file_access_compressed.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,50 +37,50 @@
class FileAccessCompressed : public FileAccess {
Compression::Mode cmode;
bool writing;
uint32_t write_pos;
uint64_t write_pos;
uint8_t *write_ptr;
uint32_t write_buffer_size;
uint32_t write_max;
uint64_t write_max;
uint32_t block_size;
mutable bool read_eof;
mutable bool at_end;

struct ReadBlock {
int csize;
int offset;
uint32_t csize;
uint64_t offset;
};

mutable Vector<uint8_t> comp_buffer;
uint8_t *read_ptr;
mutable int read_block;
int read_block_count;
mutable int read_block_size;
mutable int read_pos;
mutable uint32_t read_block;
uint32_t read_block_count;
mutable uint32_t read_block_size;
mutable uint64_t read_pos;
Vector<ReadBlock> read_blocks;
uint32_t read_total;
uint64_t read_total;

String magic;
mutable Vector<uint8_t> buffer;
FileAccess *f;

public:
void configure(const String &p_magic, Compression::Mode p_mode = Compression::MODE_ZSTD, int p_block_size = 4096);
void configure(const String &p_magic, Compression::Mode p_mode = Compression::MODE_ZSTD, uint32_t p_block_size = 4096);

Error open_after_magic(FileAccess *p_base);

virtual Error _open(const String &p_path, int p_mode_flags); ///< open a file
virtual void close(); ///< close a file
virtual bool is_open() const; ///< true when file is open

virtual void seek(size_t p_position); ///< seek to a given position
virtual void seek(uint64_t p_position); ///< seek to a given position
virtual void seek_end(int64_t p_position = 0); ///< seek from the end of file
virtual size_t get_position() const; ///< get position in the file
virtual size_t get_len() const; ///< get size of the file
virtual uint64_t get_position() const; ///< get position in the file
virtual uint64_t get_len() const; ///< get size of the file

virtual bool eof_reached() const; ///< reading passed EOF

virtual uint8_t get_8() const; ///< get a byte
virtual int get_buffer(uint8_t *p_dst, int p_length) const;
virtual uint64_t get_buffer(uint8_t *p_dst, uint64_t p_length) const;

virtual Error get_error() const; ///< get last error

Expand Down
Loading

0 comments on commit 817ffc0

Please sign in to comment.