BUG: Fix large file read/write error on Mac

apache · Nov 1, 2017 · 6ed9287 · 6ed9287
1 parent 30158ad
commit 6ed9287
Showing 1 changed file with 27 additions and 3 deletions.
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
@@ -246,7 +246,24 @@ static inline Status FileRead(int fd, uint8_t* buffer, int64_t nbytes,
   }
   *bytes_read = static_cast<int64_t>(_read(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
-  *bytes_read = static_cast<int64_t>(read(fd, buffer, static_cast<size_t>(nbytes)));
+  int64_t n = 0;
+  int64_t N = nbytes;
+  int64_t ret = 0;
+
+  *bytes_read = 0;
+
+  while (ret != -1 && n < N) {
+    int64_t i = std::min(static_cast<int64_t>(INT32_MAX), N - n);
+    ret = static_cast<int64_t>(read(fd, buffer + n, static_cast<size_t>(i)));
+
+    if (ret != -1) {
+      *bytes_read += ret;
+    } else {
+      *bytes_read = ret;
+    }
+
+    n += i;
+  }
 #endif
 
   if (*bytes_read == -1) {
@@ -258,14 +275,21 @@ static inline Status FileRead(int fd, uint8_t* buffer, int64_t nbytes,
 }
 
 static inline Status FileWrite(int fd, const uint8_t* buffer, int64_t nbytes) {
-  int ret;
+  int ret = 0;
 #if defined(_MSC_VER)
   if (nbytes > INT32_MAX) {
     return Status::IOError("Unable to write > 2GB blocks to file yet");
   }
   ret = static_cast<int>(_write(fd, buffer, static_cast<uint32_t>(nbytes)));
 #else
-  ret = static_cast<int>(write(fd, buffer, static_cast<size_t>(nbytes)));
+  int64_t n = 0;
+  int64_t N = nbytes;
+
+  while (ret != -1 && n < N) {
+    int64_t i = std::min(static_cast<int64_t>(INT32_MAX), N - n);
+    ret = static_cast<int>(write(fd, buffer + n, static_cast<size_t>(i)));
+    n += i;
+  }
 #endif
 
   if (ret == -1) {