Skip to content

Commit 34932ce

Browse files
jonsimantovwu-hui
jonsimantov
and
wu-hui
authored
Patch LevelDB so Firestore can support international paths on Windows (#11276)
* Delete invalid move tests * add some back * Patch leveldb to support UTF-16 paths on Windows. * Add patch for both 1.22 and 1.23, since C++ SDK patches to 1.23. * Fix function name in patch. * Remove extraneous override. * Add patch files to whitespace script. --------- Co-authored-by: Wu-Hui <wu.hui.github@gmail.com>
1 parent eb9c7d6 commit 34932ce

File tree

5 files changed

+568
-4
lines changed

5 files changed

+568
-4
lines changed

‎cmake/external/leveldb-1.22_windows_paths.patch

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
diff --git a/util/env_windows.cc b/util/env_windows.cc
2+
index 09e3df6..23d60f1 100644
3+
--- a/util/env_windows.cc
4+
+++ b/util/env_windows.cc
5+
@@ -362,9 +362,11 @@ class WindowsEnv : public Env {
6+
*result = nullptr;
7+
DWORD desired_access = GENERIC_READ;
8+
DWORD share_mode = FILE_SHARE_READ;
9+
- ScopedHandle handle =
10+
- ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr,
11+
- OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
12+
+ auto wFilename = toUtf16(fname);
13+
+ ScopedHandle handle = ::CreateFileW(
14+
+ wFilename.c_str(), desired_access, share_mode,
15+
+ /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
16+
+ /*hTemplateFile=*/nullptr);
17+
if (!handle.is_valid()) {
18+
return WindowsError(fname, ::GetLastError());
19+
}
20+
@@ -378,10 +380,12 @@ class WindowsEnv : public Env {
21+
DWORD desired_access = GENERIC_READ;
22+
DWORD share_mode = FILE_SHARE_READ;
23+
DWORD file_flags = FILE_ATTRIBUTE_READONLY;
24+
-
25+
+ auto wFilename = toUtf16(fname);
26+
ScopedHandle handle =
27+
- ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr,
28+
- OPEN_EXISTING, file_flags, nullptr);
29+
+ ::CreateFileW(wFilename.c_str(), desired_access, share_mode,
30+
+ /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING,
31+
+ FILE_ATTRIBUTE_READONLY,
32+
+ /*hTemplateFile=*/nullptr);
33+
if (!handle.is_valid()) {
34+
return WindowsError(fname, ::GetLastError());
35+
}
36+
@@ -396,10 +400,12 @@ class WindowsEnv : public Env {
37+
}
38+
39+
ScopedHandle mapping =
40+
- ::CreateFileMappingA(handle.get(),
41+
- /*security attributes=*/nullptr, PAGE_READONLY,
42+
- /*dwMaximumSizeHigh=*/0,
43+
- /*dwMaximumSizeLow=*/0, nullptr);
44+
+ ::CreateFileMappingW(handle.get(),
45+
+ /*security attributes=*/nullptr,
46+
+ PAGE_READONLY,
47+
+ /*dwMaximumSizeHigh=*/0,
48+
+ /*dwMaximumSizeLow=*/0,
49+
+ /*lpName=*/nullptr);
50+
if (mapping.is_valid()) {
51+
void* base = MapViewOfFile(mapping.get(), FILE_MAP_READ, 0, 0, 0);
52+
if (base) {
53+
@@ -421,10 +427,11 @@ class WindowsEnv : public Env {
54+
WritableFile** result) override {
55+
DWORD desired_access = GENERIC_WRITE;
56+
DWORD share_mode = 0;
57+
-
58+
- ScopedHandle handle =
59+
- ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr,
60+
- CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
61+
+ auto wFilename = toUtf16(fname);
62+
+ ScopedHandle handle = ::CreateFileW(
63+
+ wFilename.c_str(), desired_access, share_mode,
64+
+ /*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL,
65+
+ /*hTemplateFile=*/nullptr);
66+
if (!handle.is_valid()) {
67+
*result = nullptr;
68+
return WindowsError(fname, ::GetLastError());
69+
@@ -436,9 +443,13 @@ class WindowsEnv : public Env {
70+
71+
Status NewAppendableFile(const std::string& fname,
72+
WritableFile** result) override {
73+
- ScopedHandle handle =
74+
- ::CreateFileA(fname.c_str(), FILE_APPEND_DATA, 0, nullptr, OPEN_ALWAYS,
75+
- FILE_ATTRIBUTE_NORMAL, nullptr);
76+
+ DWORD desired_access = FILE_APPEND_DATA;
77+
+ DWORD share_mode = 0; // Exclusive access.
78+
+ auto wFilename = toUtf16(fname);
79+
+ ScopedHandle handle = ::CreateFileW(
80+
+ wFilename.c_str(), desired_access, share_mode,
81+
+ /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
82+
+ /*hTemplateFile=*/nullptr);
83+
if (!handle.is_valid()) {
84+
*result = nullptr;
85+
return WindowsError(fname, ::GetLastError());
86+
@@ -449,14 +460,16 @@ class WindowsEnv : public Env {
87+
}
88+
89+
bool FileExists(const std::string& fname) override {
90+
- return GetFileAttributesA(fname.c_str()) != INVALID_FILE_ATTRIBUTES;
91+
+ auto wFilename = toUtf16(fname);
92+
+ return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES;
93+
}
94+
95+
Status GetChildren(const std::string& dir,
96+
std::vector<std::string>* result) override {
97+
const std::string find_pattern = dir + "\\*";
98+
- WIN32_FIND_DATAA find_data;
99+
- HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data);
100+
+ WIN32_FIND_DATAW find_data;
101+
+ auto wFind_pattern = toUtf16(find_pattern);
102+
+ HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data);
103+
if (dir_handle == INVALID_HANDLE_VALUE) {
104+
DWORD last_error = ::GetLastError();
105+
if (last_error == ERROR_FILE_NOT_FOUND) {
106+
@@ -468,11 +481,12 @@ class WindowsEnv : public Env {
107+
char base_name[_MAX_FNAME];
108+
char ext[_MAX_EXT];
109+
110+
- if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name,
111+
+ auto find_data_filename = toUtf8(find_data.cFileName);
112+
+ if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, base_name,
113+
ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) {
114+
result->emplace_back(std::string(base_name) + ext);
115+
}
116+
- } while (::FindNextFileA(dir_handle, &find_data));
117+
+ } while (::FindNextFileW(dir_handle, &find_data));
118+
DWORD last_error = ::GetLastError();
119+
::FindClose(dir_handle);
120+
if (last_error != ERROR_NO_MORE_FILES) {
121+
@@ -482,13 +496,16 @@ class WindowsEnv : public Env {
122+
}
123+
124+
Status DeleteFile(const std::string& fname) override {
125+
- if (!::DeleteFileA(fname.c_str())) {
126+
+ auto wFilename = toUtf16(fname);
127+
+ if (!::DeleteFileW(wFilename.c_str())) {
128+
return WindowsError(fname, ::GetLastError());
129+
}
130+
return Status::OK();
131+
}
132+
133+
Status CreateDir(const std::string& name) override {
134+
+ auto wDirname = toUtf16(name);
135+
+ if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) {
136+
if (!::CreateDirectoryA(name.c_str(), nullptr)) {
137+
return WindowsError(name, ::GetLastError());
138+
}
139+
@@ -496,7 +513,8 @@ class WindowsEnv : public Env {
140+
}
141+
142+
Status DeleteDir(const std::string& name) override {
143+
- if (!::RemoveDirectoryA(name.c_str())) {
144+
+ auto wDirname = toUtf16(name);
145+
+ if (!::RemoveDirectoryW(wDirname.c_str())) {
146+
return WindowsError(name, ::GetLastError());
147+
}
148+
return Status::OK();
149+
@@ -504,7 +522,9 @@ class WindowsEnv : public Env {
150+
151+
Status GetFileSize(const std::string& fname, uint64_t* size) override {
152+
WIN32_FILE_ATTRIBUTE_DATA attrs;
153+
- if (!::GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) {
154+
+ auto wFilename = toUtf16(filename);
155+
+ if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard,
156+
+ &attrs)) {
157+
return WindowsError(fname, ::GetLastError());
158+
}
159+
ULARGE_INTEGER file_size;
160+
@@ -518,7 +538,9 @@ class WindowsEnv : public Env {
161+
const std::string& target) override {
162+
// Try a simple move first. It will only succeed when |to_path| doesn't
163+
// already exist.
164+
- if (::MoveFileA(src.c_str(), target.c_str())) {
165+
+ auto wFrom = toUtf16(src);
166+
+ auto wTo = toUtf16(target);
167+
+ if (::MoveFileW(wFrom.c_str(), wTo.c_str())) {
168+
return Status::OK();
169+
}
170+
DWORD move_error = ::GetLastError();
171+
@@ -527,7 +549,7 @@ class WindowsEnv : public Env {
172+
// succeed when |to_path| does exist. When writing to a network share, we
173+
// may not be able to change the ACLs. Ignore ACL errors then
174+
// (REPLACEFILE_IGNORE_MERGE_ERRORS).
175+
- if (::ReplaceFileA(target.c_str(), src.c_str(), nullptr,
176+
+ if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr,
177+
REPLACEFILE_IGNORE_MERGE_ERRORS, nullptr, nullptr)) {
178+
return Status::OK();
179+
}
180+
@@ -546,8 +568,9 @@ class WindowsEnv : public Env {
181+
Status LockFile(const std::string& fname, FileLock** lock) override {
182+
*lock = nullptr;
183+
Status result;
184+
- ScopedHandle handle = ::CreateFileA(
185+
- fname.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
186+
+ auto wFilename = toUtf16(fname);
187+
+ ScopedHandle handle = ::CreateFileW(
188+
+ wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ,
189+
/*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL,
190+
nullptr);
191+
if (!handle.is_valid()) {
192+
@@ -584,10 +607,11 @@ class WindowsEnv : public Env {
193+
return Status::OK();
194+
}
195+
196+
- char tmp_path[MAX_PATH];
197+
- if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) {
198+
+ wchar_t wtmp_path[MAX_PATH];
199+
+ if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) {
200+
return WindowsError("GetTempPath", ::GetLastError());
201+
}
202+
+ std::string tmp_path = toUtf8(std::wstring(wtmp_path));
203+
std::stringstream ss;
204+
ss << tmp_path << "leveldbtest-" << std::this_thread::get_id();
205+
*result = ss.str();
206+
@@ -598,7 +622,8 @@ class WindowsEnv : public Env {
207+
}
208+
209+
Status NewLogger(const std::string& filename, Logger** result) override {
210+
- std::FILE* fp = std::fopen(filename.c_str(), "w");
211+
+ auto wFilename = toUtf16(filename);
212+
+ std::FILE* fp = _wfopen(wFilename.c_str(), L"w");
213+
if (fp == nullptr) {
214+
*result = nullptr;
215+
return WindowsError("NewLogger", ::GetLastError());
216+
@@ -640,6 +665,31 @@ class WindowsEnv : public Env {
217+
bool started_bgthread_;
218+
std::deque<BGItem> queue_;
219+
Limiter mmap_limiter_;
220+
+
221+
+ // Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string.
222+
+ // See http://utf8everywhere.org/#windows
223+
+ std::string toUtf8(const std::wstring& wstr) {
224+
+ if (wstr.empty()) return std::string();
225+
+ int size_needed = WideCharToMultiByte(
226+
+ CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
227+
+ std::string strTo(size_needed, 0);
228+
+ WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0],
229+
+ size_needed, NULL, NULL);
230+
+ return strTo;
231+
+ }
232+
+
233+
+ // Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character
234+
+ // string.
235+
+ // See http://utf8everywhere.org/#windows
236+
+ std::wstring toUtf16(const std::string& str) {
237+
+ if (str.empty()) return std::wstring();
238+
+ int size_needed = MultiByteToWideChar(
239+
+ CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0);
240+
+ std::wstring strTo(size_needed, 0);
241+
+ MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0],
242+
+ size_needed);
243+
+ return strTo;
244+
+ }
245+
};
246+
247+
// Return the maximum number of concurrent mmaps.

0 commit comments

Comments
 (0)