Fix encoding bugs

An ~advanced~ static analysis technique helped me find many places, where we convert std::string to std::filesystem::path without going through StrUtils::ToString(). Most of this are bugs: std::filesystem::path interprets string in the "native narrow encoding" instead of utf-8. Here is my technique: ```diff *** /usr/include/c++/14/bits/fs_path.h.backup Thu Aug 22 08:31:54 2024 --- /usr/include/c++/14/bits/fs_path.h Thu Aug 22 09:36:18 2024 *************** *** 321,326 **** --- 321,330 ---- _M_cmpts(std::move(__p._M_cmpts)) { __p.clear(); } + [[deprecated]] + path(const string_type& __source, format = auto_format) {} + + [[deprecated]] path(string_type&& __source, format = auto_format) : _M_pathname(std::move(__source)) { _M_split_cmpts(); } *************** *** 358,363 **** --- 362,369 ---- path& operator=(const path&); path& operator=(path&&) noexcept; + + [[deprecated]] path& operator=(string_type&& __source); path& assign(string_type&& __source); *************** *** 380,385 **** --- 386,394 ---- path& operator/=(const path& __p); + [[deprecated]] + path& operator/=(const string_type& __p) {} + template<typename _Source> __detail::_Path<_Source>& operator/=(_Source const& __source) *************** *** 407,415 **** --- 416,428 ---- // concatenation path& operator+=(const path& __x); + + [[deprecated]] path& operator+=(const string_type& __x); path& operator+=(const value_type* __x); path& operator+=(value_type __x); + + [[deprecated]] path& operator+=(basic_string_view<value_type> __x); template<typename _Source> *************** *** 451,456 **** --- 464,471 ---- const string_type& native() const noexcept { return _M_pathname; } const value_type* c_str() const noexcept { return _M_pathname.c_str(); } + + [[deprecated]] operator string_type() const { return _M_pathname; } template<typename _CharT, typename _Traits = std::char_traits<_CharT>, ```
hexagonrecursion · Aug 22, 2024 · 02492ff · 02492ff
1 parent 172be45
commit 02492ff
Show file tree

Hide file tree

Showing 18 changed files with 67 additions and 57 deletions.
diff --git a/colobot-base/src/common/font_loader.cpp b/colobot-base/src/common/font_loader.cpp
@@ -85,5 +85,5 @@ std::optional<std::filesystem::path> CFontLoader::GetFont(Gfx::FontType type) co
     if (iterator == m_fonts.end())
         return std::nullopt;
     else
-        return std::filesystem::path("fonts") / iterator->second;
+        return "fonts" / TempToPath(iterator->second);
 }
diff --git a/colobot-base/src/graphics/engine/oldmodelmanager.cpp b/colobot-base/src/graphics/engine/oldmodelmanager.cpp
@@ -51,13 +51,13 @@ bool COldModelManager::LoadModel(const std::string& name, bool mirrored, int tea
     std::unique_ptr<CModel> model;
     try
     {
-        auto extension = std::filesystem::path(name).extension().string();
+        auto extension = TempToPath(name).extension().string();
 
         if (!extension.empty())
         {
             GetLogger()->Debug("Loading model '%%'", name);
 
-            model = ModelInput::Read("models/" + name);
+            model = ModelInput::Read(TempToPath("models/" + name));
 
             if (model->GetMeshCount() == 0)
                 return false;
@@ -67,23 +67,23 @@ bool COldModelManager::LoadModel(const std::string& name, bool mirrored, int tea
 
         auto gltf_path = "models/" + name + ".gltf";
 
-        if (CResourceManager::Exists(gltf_path))
+        if (CResourceManager::Exists(TempToPath(gltf_path)))
         {
             GetLogger()->Debug("Loading model '%%'", (name + ".gltf"));
 
-            model = ModelInput::Read(gltf_path);
+            model = ModelInput::Read(TempToPath(gltf_path));
 
             if (model->GetMeshCount() > 0)
                 goto skip;
         }
 
         auto mod_path = "models/" + name + ".mod";
 
-        if (CResourceManager::Exists(mod_path))
+        if (CResourceManager::Exists(TempToPath(mod_path)))
         {
             GetLogger()->Debug("Loading model '%%'", (name + ".mod"));
 
-            model = ModelInput::Read(mod_path);
+            model = ModelInput::Read(TempToPath(mod_path));
 
             if (model->GetMeshCount() > 0)
                 goto skip;

diff --git a/colobot-base/src/graphics/engine/particle.cpp b/colobot-base/src/graphics/engine/particle.cpp
@@ -3458,7 +3458,7 @@ void CParticle::DrawParticle(int sheet)
             {
                 std::string name;
                 NameParticle(name, t);
-                auto texture = m_engine->LoadTexture(std::filesystem::path("textures") / name);
+                auto texture = m_engine->LoadTexture("textures" / TempToPath(name));
                 m_renderer->SetTexture(texture);
                 loadTexture = true;
             }

diff --git a/colobot-base/src/graphics/model/model_gltf.cpp b/colobot-base/src/graphics/model/model_gltf.cpp
@@ -145,7 +145,7 @@ void GLTFLoader::ReadBuffers()
 
         if (node.contains("uri"))
         {
-            auto uri = m_directory / node["uri"].get<std::string>();
+            auto uri = m_directory / StrUtils::ToPath(node["uri"].get<std::string>());
 
             CInputStream stream(uri);
 

diff --git a/colobot-base/src/graphics/model/model_manager.cpp b/colobot-base/src/graphics/model/model_manager.cpp
@@ -20,6 +20,7 @@
 #include "graphics/model/model_manager.h"
 
 #include "common/logger.h"
+#include "common/stringutils.h"
 
 #include "common/resources/inputstream.h"
 
@@ -35,7 +36,7 @@ CModel* CModelManager::GetModel(const std::string& modelName)
     if (it != m_models.end())
         return it->second.get();
 
-    std::filesystem::path modelFile = "models-new/" + modelName + ".txt";
+    std::filesystem::path modelFile = TempToPath("models-new/" + modelName + ".txt");
 
     GetLogger()->Debug("Loading new model: %%", modelFile.string());
 

diff --git a/colobot-base/src/level/parser/parser.cpp b/colobot-base/src/level/parser/parser.cpp
@@ -89,7 +89,7 @@ std::filesystem::path CLevelParser::BuildScenePath(std::string_view category, in
 
     if (category == "custom")
     {
-        path /= CRobotMain::GetInstancePointer()->GetCustomLevelName(chapter);
+        path /= TempToPath(CRobotMain::GetInstancePointer()->GetCustomLevelName(chapter));
 
         if (rank == 0)
         {
@@ -100,7 +100,7 @@ std::filesystem::path CLevelParser::BuildScenePath(std::string_view category, in
         }
         else
         {
-            path /= StrUtils::Format("level%03d", rank);
+            path /= StrUtils::ToPath(StrUtils::Format("level%03d", rank));
 
             if (sceneFile)
             {
@@ -117,11 +117,11 @@ std::filesystem::path CLevelParser::BuildScenePath(std::string_view category, in
     else if (category == "win" || category == "lost")
     {
         assert(chapter == 0);
-        path /= StrUtils::Format("%s%03d.txt", category, rank);
+        path /= StrUtils::ToPath(StrUtils::Format("%s%03d.txt", category, rank));
     }
     else
     {
-        path /= StrUtils::Format("chapter%03d", chapter);
+        path /= StrUtils::ToPath(StrUtils::Format("chapter%03d", chapter));
         if (rank == 000)
         {
             if (sceneFile)
@@ -131,7 +131,7 @@ std::filesystem::path CLevelParser::BuildScenePath(std::string_view category, in
         }
         else
         {
-            path /= StrUtils::Format("level%03d", rank);
+            path /= StrUtils::ToPath(StrUtils::Format("level%03d", rank));
             if (sceneFile)
             {
                 path /= "scene.txt";
@@ -322,7 +322,7 @@ std::filesystem::path CLevelParser::InjectLevelPaths(const std::filesystem::path
     if(!m_pathLvl.empty() ) newPath = StrUtils::Replace(newPath, "%lvl%",  StrUtils::ToString(m_pathLvl));
     if(!m_pathChap.empty()) newPath = StrUtils::Replace(newPath, "%chap%", StrUtils::ToString(m_pathChap));
     if(!m_pathCat.empty() ) newPath = StrUtils::Replace(newPath, "%cat%",  StrUtils::ToString(m_pathCat));
-    if(newPath == path && !path.empty())
+    if(newPath == StrUtils::ToString(path) && !path.empty())
     {
         newPath = StrUtils::ToString(defaultDir) + (!defaultDir.empty() ? "/" : "") + newPath;
     }

diff --git a/colobot-base/src/level/parser/parserline.cpp b/colobot-base/src/level/parser/parserline.cpp
@@ -50,7 +50,7 @@ void CLevelParserLine::SetLevel(CLevelParser* level)
     // Only on the first call - this makes sure the level name doesn't change if the file is loaded using #Include
     if (m_levelFilename.empty())
     {
-        m_levelFilename = StrUtils::ToString(m_level->GetFilename());
+        m_levelFilename = m_level->GetFilename();
     }
 }
 

diff --git a/colobot-base/src/level/player_profile.cpp b/colobot-base/src/level/player_profile.cpp
@@ -257,7 +257,7 @@ int CPlayerProfile::GetSelectedRank(LevelCategory category)
 void CPlayerProfile::LoadFinishedLevels(LevelCategory category)
 {
     m_levelInfo[category].clear();
-    std::filesystem::path filename = GetSaveFile(GetLevelCategoryDir(category) + ".gam");
+    std::filesystem::path filename = GetSaveFile(TempToPath(GetLevelCategoryDir(category) + ".gam"));
 
     if (!CResourceManager::Exists(filename))
         return;
@@ -299,7 +299,7 @@ void CPlayerProfile::LoadFinishedLevels(LevelCategory category)
 
 void CPlayerProfile::SaveFinishedLevels(LevelCategory category)
 {
-    std::filesystem::path filename = GetSaveFile(GetLevelCategoryDir(category) + ".gam");
+    std::filesystem::path filename = GetSaveFile(TempToPath(GetLevelCategoryDir(category) + ".gam"));
     COutputStream file;
     file.open(filename);
     if (!file.is_open())
@@ -415,7 +415,7 @@ void CPlayerProfile::LoadAppearance()
 
     try
     {
-        CLevelParser appearanceParser(StrUtils::ToString(filename));
+        CLevelParser appearanceParser(filename);
         appearanceParser.Load();
         CLevelParserLine* line;
 
@@ -438,7 +438,7 @@ void CPlayerProfile::SaveAppearance()
 {
     try
     {
-        CLevelParser appearanceParser(StrUtils::ToString(GetSaveFile("face.gam")));
+        CLevelParser appearanceParser(GetSaveFile("face.gam"));
         CLevelParserLineUPtr line;
 
         line = std::make_unique<CLevelParserLine>("Head");
@@ -467,7 +467,7 @@ bool CPlayerProfile::HasAnySavedScene()
     auto saveDirs = CResourceManager::ListDirectories(GetSaveDir());
     for (auto dir : saveDirs)
     {
-        if (CResourceManager::Exists(GetSaveFile(StrUtils::ToString(dir / "data.sav"))))
+        if (CResourceManager::Exists(GetSaveFile(dir / "data.sav")))
         {
             return true;
         }
@@ -485,7 +485,7 @@ std::vector<SavedScene> CPlayerProfile::GetSavedSceneList()
         std::filesystem::path savegameFile = GetSaveFile(dir / "data.sav");
         if (CResourceManager::Exists(savegameFile) && CResourceManager::GetFileSize(savegameFile) > 0)
         {
-            CLevelParser levelParser(StrUtils::ToString(savegameFile));
+            CLevelParser levelParser(savegameFile);
             levelParser.Load();
             CLevelParserLine* line = levelParser.GetIfDefined("Created");
             int time = line != nullptr ? line->GetParam("date")->AsInt() : 0;

diff --git a/colobot-base/src/level/robotmain.cpp b/colobot-base/src/level/robotmain.cpp
@@ -599,7 +599,7 @@ void CRobotMain::ChangePhase(Phase phase)
                     pe->SetFontType(Gfx::FONT_COMMON);
                     pe->SetEditCap(false);
                     pe->SetHighlightCap(false);
-                    pe->ReadText(std::string("help/") + m_app->GetLanguageChar() + std::string("/win.txt"));
+                    pe->ReadText(TempToPath(std::string("help/") + m_app->GetLanguageChar() + std::string("/win.txt")));
                 }
                 else
                 {
@@ -2249,7 +2249,7 @@ void CRobotMain::HelpObject()
     std::string filename = GetHelpFilename(obj->GetType());
     if (filename.empty()) return;
 
-    StartDisplayInfo(filename, -1);
+    StartDisplayInfo(TempToPath(filename), -1);
 }
 
 
@@ -2873,7 +2873,7 @@ void CRobotMain::CreateScene(bool soluce, bool fixScene, bool resetObject)
 
             if (line->GetCommand() == "ScriptFile" && !resetObject)
             {
-                m_scriptFile = line->GetParam("name")->AsString();
+                m_scriptFile = TempToPath(line->GetParam("name")->AsString());
                 continue;
             }
 
@@ -2925,7 +2925,7 @@ void CRobotMain::CreateScene(bool soluce, bool fixScene, bool resetObject)
                                 GetLogger()->Warn("This level is using deprecated way of defining %% scene. Please change the %%= parameter in EndingFile from %% to \"levels/other/%1$s%2$03d.txt\".\n", type, type, rank);
                                 std::stringstream ss;
                                 ss << std::setfill('0') << std::setw(3) << rank << ".txt";
-                                return std::filesystem::path("levels/other") / type / ss.str();
+                                return "levels/other" / StrUtils::ToPath(type) / StrUtils::ToPath(ss.str());
                             }
                             else
                             {
@@ -3463,7 +3463,7 @@ void CRobotMain::CreateScene(bool soluce, bool fixScene, bool resetObject)
                         char categoryChar = GetLevelCategoryDir(m_levelCategory)[0];
                         programStorage->LoadAllProgramsForLevel(
                             line.get(),
-                            StrUtils::ToString(m_playerProfile->GetSaveFile(StrUtils::Format("%c%.3d%.3d", categoryChar, m_levelChap, m_levelRank))),
+                            StrUtils::ToString(m_playerProfile->GetSaveFile(StrUtils::ToPath(StrUtils::Format("%c%.3d%.3d", categoryChar, m_levelChap, m_levelRank)))),
                             soluce
                         );
                     }
@@ -4329,7 +4329,7 @@ void CRobotMain::SaveOneScript(CObject *obj)
     CProgramStorageObject* programStorage = dynamic_cast<CProgramStorageObject*>(obj);
 
     char categoryChar = GetLevelCategoryDir(m_levelCategory)[0];
-    programStorage->SaveAllUserPrograms(StrUtils::ToString(m_playerProfile->GetSaveFile(StrUtils::Format("%c%.3d%.3d", categoryChar, m_levelChap, m_levelRank))));
+    programStorage->SaveAllUserPrograms(StrUtils::ToString(m_playerProfile->GetSaveFile(StrUtils::ToPath(StrUtils::Format("%c%.3d%.3d", categoryChar, m_levelChap, m_levelRank)))));
 }
 
 //! Saves the stack of the program in execution of a robot
@@ -5330,7 +5330,7 @@ void CRobotMain::SetLevel(LevelCategory cat, int chap, int rank)
     m_levelCategory = cat;
     m_levelChap = chap;
     m_levelRank = rank;
-    m_levelFile = StrUtils::ToString(CLevelParser::BuildScenePath(m_levelCategory, m_levelChap, m_levelRank));
+    m_levelFile = CLevelParser::BuildScenePath(m_levelCategory, m_levelChap, m_levelRank);
 }
 
 LevelCategory CRobotMain::GetLevelCategory()
@@ -5664,7 +5664,7 @@ void CRobotMain::Autosave()
     strftime(timestr, 99, "%y%m%d%H%M%S", localtime(&now));
     strftime(infostr, 99, "%y.%m.%d %H:%M", localtime(&now));
     std::string info = std::string("[AUTOSAVE] ") + infostr;
-    std::filesystem::path dir = m_playerProfile->GetSaveFile(std::string("autosave") + timestr);
+    std::filesystem::path dir = m_playerProfile->GetSaveFile(StrUtils::ToPath(std::string("autosave") + timestr));
 
     m_playerProfile->SaveScene(dir, info);
 }
@@ -5696,7 +5696,7 @@ void CRobotMain::QuickLoad()
 
 void CRobotMain::LoadSaveFromDirName(const std::string& gameDir)
 {
-    std::filesystem::path dir = m_playerProfile->GetSaveFile(gameDir);
+    std::filesystem::path dir = m_playerProfile->GetSaveFile(TempToPath(gameDir));
     if(!CResourceManager::Exists(dir))
     {
         GetLogger()->Error("Save slot not found");

diff --git a/colobot-base/src/object/auto/autoegg.cpp b/colobot-base/src/object/auto/autoegg.cpp
@@ -196,7 +196,7 @@ bool CAutoEgg::EventProcess(const Event &event)
 
             CProgramStorageObject* programStorage = dynamic_cast<CProgramStorageObject*>(alien);
             Program* program = programStorage->AddProgram();
-            programStorage->ReadProgram(program, InjectLevelPathsForCurrentLevel(m_alienProgramName, "ai"));
+            programStorage->ReadProgram(program, InjectLevelPathsForCurrentLevel(TempToPath(m_alienProgramName), "ai"));
             program->readOnly = true;
             program->filename = m_alienProgramName;
             programmable->RunProgram(program);

diff --git a/colobot-base/src/object/auto/autofactory.cpp b/colobot-base/src/object/auto/autofactory.cpp
@@ -678,7 +678,7 @@ bool CAutoFactory::CreateVehicle()
         for (const std::string& name : m_main->GetNewScriptNames(m_type))
         {
             Program* prog = programStorage->AddProgram();
-            programStorage->ReadProgram(prog, InjectLevelPathsForCurrentLevel(name, "ai"));
+            programStorage->ReadProgram(prog, InjectLevelPathsForCurrentLevel(TempToPath(name), "ai"));
             prog->readOnly = true;
             prog->filename = name;
         }

diff --git a/colobot-base/src/script/scriptfunc.cpp b/colobot-base/src/script/scriptfunc.cpp
@@ -1709,7 +1709,7 @@ bool CScriptFunctions::rProduce(CBotVar* var, CBotVar* result, int& exception, v
 
     if (!name.empty())
     {
-        std::filesystem::path name2 = InjectLevelPathsForCurrentLevel(name, "ai");
+        std::filesystem::path name2 = InjectLevelPathsForCurrentLevel(TempToPath(name), "ai");
         if (object->Implements(ObjectInterfaceType::Programmable))
         {
             CProgramStorageObject* programStorage = dynamic_cast<CProgramStorageObject*>(object);
@@ -3466,7 +3466,7 @@ class CBotFileAccessHandlerColobot : public CBotFileAccessHandler
     {
         std::string fname = PrepareFilename(filename);
         GetLogger()->Info("CBot delete file '%%'", fname);
-        return CResourceManager::Remove(fname);
+        return CResourceManager::Remove(TempToPath(fname));
     }
 
 private:

diff --git a/colobot-base/src/ui/controls/edit.cpp b/colobot-base/src/ui/controls/edit.cpp
@@ -818,10 +818,10 @@ void CEdit::HyperJump(std::string name, std::string marker)
     }
 
     std::string filename = name + std::string(".txt");
-    filename = StrUtils::ToString(InjectLevelPathsForCurrentLevel(filename, "help/%lng%"));
+    filename = StrUtils::ToString(InjectLevelPathsForCurrentLevel(TempToPath(filename), "help/%lng%"));
     filename = StrUtils::Replace(filename, "\\", "/"); //TODO: Fix this in files
 
-    if ( ReadText(filename) )
+    if ( ReadText(TempToPath(filename)) )
     {
         Justif();
 
@@ -903,7 +903,7 @@ bool CEdit::HyperGo(EventType event)
         m_historyCurrent ++;
     }
 
-    ReadText(m_history[m_historyCurrent].filename);
+    ReadText(TempToPath(m_history[m_historyCurrent].filename));
     Justif();
     SetFirstLine(m_history[m_historyCurrent].firstLine);
     return true;
@@ -1170,7 +1170,7 @@ static std::string PrepareImageFilename(std::string name)
 {
     std::string filename;
     filename = name + ".png";
-    filename = StrUtils::ToString(InjectLevelPathsForCurrentLevel(filename, "icons"));
+    filename = StrUtils::ToString(InjectLevelPathsForCurrentLevel(TempToPath(filename), "icons"));
     filename = StrUtils::Replace(filename, "\\", "/"); // TODO: Fix this in files
     return filename;
 }