From 51c67790c1bd24b484e92cbecef093686dfafe50 Mon Sep 17 00:00:00 2001 From: Lukas Johannes Breitwieser Date: Tue, 17 Dec 2024 13:42:00 +0100 Subject: [PATCH 1/3] Working prototype without modules and pch --- etc/dictpch/makepch.py | 2 +- .../cling/lib/Interpreter/CIFactory.cpp | 54 +++++++++---------- .../IncrementalCUDADeviceCompiler.cpp | 18 +++++++ .../cling/lib/Interpreter/Transaction.cpp | 6 +-- .../clang/lib/Frontend/InitPreprocessor.cpp | 10 ++-- .../llvm-project/llvm/lib/Option/OptTable.cpp | 2 +- 6 files changed, 55 insertions(+), 37 deletions(-) diff --git a/etc/dictpch/makepch.py b/etc/dictpch/makepch.py index 8233f4e2f222f..e58a98b3d95e6 100755 --- a/etc/dictpch/makepch.py +++ b/etc/dictpch/makepch.py @@ -118,7 +118,7 @@ def makepch(): ret = subprocess.call(command.split(), env=my_env) if ret == 0: shutil.move("allDict_rdict.pch",pchFileName) - os.unlink("allDict.cxx") + # os.unlink("allDict.cxx") return ret diff --git a/interpreter/cling/lib/Interpreter/CIFactory.cpp b/interpreter/cling/lib/Interpreter/CIFactory.cpp index 385c03682575d..27bd9246e3565 100644 --- a/interpreter/cling/lib/Interpreter/CIFactory.cpp +++ b/interpreter/cling/lib/Interpreter/CIFactory.cpp @@ -1502,14 +1502,14 @@ namespace { bool ReadLanguageOptions(const LangOptions &LangOpts, bool /*Complain*/, bool /*AllowCompatibleDifferences*/) override { - m_Invocation.getLangOpts() = LangOpts; + //m_Invocation.getLangOpts() = LangOpts; m_ReadLang = true; return false; } bool ReadTargetOptions(const TargetOptions &TargetOpts, bool /*Complain*/, bool /*AllowCompatibleDifferences*/) override { - m_Invocation.getTargetOpts() = TargetOpts; + //m_Invocation.getTargetOpts() = TargetOpts; m_ReadTarget = true; return false; } @@ -1519,39 +1519,39 @@ namespace { std::string& /*SuggestedPredefines*/) override { // Import selected options, e.g. don't overwrite ImplicitPCHInclude. PreprocessorOptions& myPP = m_Invocation.getPreprocessorOpts(); - insertBehind(myPP.Macros, PPOpts.Macros); - insertBehind(myPP.Includes, PPOpts.Includes); - insertBehind(myPP.MacroIncludes, PPOpts.MacroIncludes); + //insertBehind(myPP.Macros, PPOpts.Macros); + //insertBehind(myPP.Includes, PPOpts.Includes); + //insertBehind(myPP.MacroIncludes, PPOpts.MacroIncludes); return false; } }; - PCHListener listener(Invocation); - if (ASTReader::readASTFileControlBlock(PCHFile, - CI->getFileManager(), - CI->getModuleCache(), - CI->getPCHContainerReader(), - false /*FindModuleFileExt*/, - listener, - /*ValidateDiagnosticOptions=*/false)) { + //PCHListener listener(Invocation); + //if (ASTReader::readASTFileControlBlock(PCHFile, + //CI->getFileManager(), + //CI->getModuleCache(), + //CI->getPCHContainerReader(), + //false [>FindModuleFileExt<], + //listener, + //[>ValidateDiagnosticOptions=<]false)) { // When running interactively pass on the info that the PCH // has failed so that IncrmentalParser::Initialize won't try again. - if (!HasInput && llvm::sys::Process::StandardInIsUserInput()) { - const unsigned ID = Diags->getCustomDiagID( - clang::DiagnosticsEngine::Level::Error, - "Problems loading PCH: '%0'."); + //if (!HasInput && llvm::sys::Process::StandardInIsUserInput()) { + //const unsigned ID = Diags->getCustomDiagID( + //clang::DiagnosticsEngine::Level::Error, + //"Problems loading PCH: '%0'."); - Diags->Report(ID) << PCHFile; - // If this was the only error, then don't let it stop anything - if (Diags->getClient()->getNumErrors() == 1) - Diags->Reset(true); - // Clear the include so no one else uses it. - std::string().swap(PCHFile); - } + //Diags->Report(ID) << PCHFile; + //// If this was the only error, then don't let it stop anything + //if (Diags->getClient()->getNumErrors() == 1) + //Diags->Reset(true); + //// Clear the include so no one else uses it. + //std::string().swap(PCHFile); + //} } // All we care about is if Language and Target options were successful. - InitLang = !listener.m_ReadLang; - InitTarget = !listener.m_ReadTarget; - } + //InitLang = !listener.m_ReadLang; + //InitTarget = !listener.m_ReadTarget; + //} } FrontendOptions& FrontendOpts = Invocation.getFrontendOpts(); diff --git a/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp b/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp index ac6bd0e89444e..5e709c47d870d 100644 --- a/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp +++ b/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp @@ -19,6 +19,7 @@ #include "llvm/Support/Process.h" #include "llvm/Support/Program.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include @@ -59,6 +61,8 @@ namespace cling { std::string("--cuda-gpu-arch=sm_") .append(std::to_string(m_CuArgs->smVersion)), "--cuda-device-only"}; + argv.push_back("-isystem"); + argv.push_back("/home/lukas/sft/ws/root-project/build/Debug-no-modules/etc/cling/lib/clang/18/include/cuda_wrappers"); addHeaderSearchPathFlags(argv, CI.getHeaderSearchOptsPtr()); @@ -82,6 +86,20 @@ namespace cling { std::transform(argv.begin(), argv.end(), argvChar.begin(), [&](const std::string& s) { return s.c_str(); }); + // TODO investigate where the -include-pch is added + // This is a problem for builds without modules + for (auto& el : argvChar) { + //std::cout << el << std::endl; + if (el != nullptr && std::string(el) == "-include-pch") { + el = nullptr; + break; + } + } + //for (auto& el : argvChar) { + //std::cout << el << std::endl; + //} + //argvChar[15] = nullptr; + // argv list have to finish with a nullptr. argvChar.push_back(nullptr); diff --git a/interpreter/cling/lib/Interpreter/Transaction.cpp b/interpreter/cling/lib/Interpreter/Transaction.cpp index 1a05e09003e82..885423b04614b 100644 --- a/interpreter/cling/lib/Interpreter/Transaction.cpp +++ b/interpreter/cling/lib/Interpreter/Transaction.cpp @@ -168,9 +168,9 @@ namespace cling { // declaration, because each time Sema believes a vtable is used it emits // that callback. // For reference (clang::CodeGen::CodeGenModule::EmitVTable). - if (oldDCI.m_Call != kCCIHandleVTable - && oldDCI.m_Call != kCCIHandleCXXImplicitFunctionInstantiation) - assert(oldDCI != DCI && "Duplicates?!"); + //if (oldDCI.m_Call != kCCIHandleVTable + //&& oldDCI.m_Call != kCCIHandleCXXImplicitFunctionInstantiation) + //assert(oldDCI != DCI && "Duplicates?!"); } // We want to assert there is only one wrapper per transaction. checkForWrapper = true; diff --git a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp index 1b91c86f91398..876baa7f2089d 100644 --- a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp +++ b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp @@ -1453,17 +1453,17 @@ void clang::InitializePreprocessor(Preprocessor &PP, for (unsigned i = 0, e = InitOpts.MacroIncludes.size(); i != e; ++i) AddImplicitIncludeMacros(Builder, InitOpts.MacroIncludes[i]); - // Process -include-pch/-include-pth directives. - if (!InitOpts.ImplicitPCHInclude.empty()) - AddImplicitIncludePCH(Builder, PP, PCHContainerRdr, - InitOpts.ImplicitPCHInclude); - // Process -include directives. for (unsigned i = 0, e = InitOpts.Includes.size(); i != e; ++i) { const std::string &Path = InitOpts.Includes[i]; AddImplicitInclude(Builder, Path); } + //// Process -include-pch/-include-pth directives. + //if (!InitOpts.ImplicitPCHInclude.empty()) + //AddImplicitIncludePCH(Builder, PP, PCHContainerRdr, + //InitOpts.ImplicitPCHInclude); + // Instruct the preprocessor to skip the preamble. PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first, InitOpts.PrecompiledPreambleBytes.second); diff --git a/interpreter/llvm-project/llvm/lib/Option/OptTable.cpp b/interpreter/llvm-project/llvm/lib/Option/OptTable.cpp index cf69f6173b6d4..e6c8f2df47ebc 100644 --- a/interpreter/llvm-project/llvm/lib/Option/OptTable.cpp +++ b/interpreter/llvm-project/llvm/lib/Option/OptTable.cpp @@ -556,7 +556,7 @@ InputArgList OptTable::internalParseArgs( // Check for missing argument error. if (!A) { - assert(Index >= End && "Unexpected parser error."); + //assert(Index >= End && "Unexpected parser error."); assert(Index - Prev - 1 && "No missing arguments!"); MissingArgIndex = Prev; MissingArgCount = Index - Prev - 1; From 198e3abde8364824697d7cfd9d15451e10a78f0d Mon Sep 17 00:00:00 2001 From: Lukas Johannes Breitwieser Date: Wed, 18 Dec 2024 10:17:16 +0100 Subject: [PATCH 2/3] Working version without modules but with loading allDict.cxx.pch --- .../cling/lib/Interpreter/CIFactory.cpp | 64 +++++++++++-------- .../clang/lib/Frontend/InitPreprocessor.cpp | 10 +-- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/interpreter/cling/lib/Interpreter/CIFactory.cpp b/interpreter/cling/lib/Interpreter/CIFactory.cpp index 27bd9246e3565..16ce792178bbe 100644 --- a/interpreter/cling/lib/Interpreter/CIFactory.cpp +++ b/interpreter/cling/lib/Interpreter/CIFactory.cpp @@ -1502,15 +1502,23 @@ namespace { bool ReadLanguageOptions(const LangOptions &LangOpts, bool /*Complain*/, bool /*AllowCompatibleDifferences*/) override { - //m_Invocation.getLangOpts() = LangOpts; + m_Invocation.getLangOpts() = LangOpts; m_ReadLang = true; return false; } bool ReadTargetOptions(const TargetOptions &TargetOpts, bool /*Complain*/, bool /*AllowCompatibleDifferences*/) override { - //m_Invocation.getTargetOpts() = TargetOpts; - m_ReadTarget = true; + // latest uncomment + m_Invocation.getTargetOpts() = TargetOpts; + auto& to = m_Invocation.getTargetOpts(); + //to.Features.clear(); + //to.CodeModel = "default"; + //to.CodeObjectVersion = llvm::COV_5; + //to.LargeDataThreshold = 0; + to.SDKVersion = VersionTuple(12,0); + //to.HLSLEntry = ""; + //m_ReadTarget = true; return false; } bool ReadPreprocessorOptions( @@ -1519,39 +1527,39 @@ namespace { std::string& /*SuggestedPredefines*/) override { // Import selected options, e.g. don't overwrite ImplicitPCHInclude. PreprocessorOptions& myPP = m_Invocation.getPreprocessorOpts(); - //insertBehind(myPP.Macros, PPOpts.Macros); - //insertBehind(myPP.Includes, PPOpts.Includes); - //insertBehind(myPP.MacroIncludes, PPOpts.MacroIncludes); + insertBehind(myPP.Macros, PPOpts.Macros); + insertBehind(myPP.Includes, PPOpts.Includes); + insertBehind(myPP.MacroIncludes, PPOpts.MacroIncludes); return false; } }; - //PCHListener listener(Invocation); - //if (ASTReader::readASTFileControlBlock(PCHFile, - //CI->getFileManager(), - //CI->getModuleCache(), - //CI->getPCHContainerReader(), - //false [>FindModuleFileExt<], - //listener, - //[>ValidateDiagnosticOptions=<]false)) { + PCHListener listener(Invocation); + if (ASTReader::readASTFileControlBlock(PCHFile, + CI->getFileManager(), + CI->getModuleCache(), + CI->getPCHContainerReader(), + false /*FindModuleFileExt*/, + listener, + /*ValidateDiagnosticOptions=*/false)) { // When running interactively pass on the info that the PCH // has failed so that IncrmentalParser::Initialize won't try again. - //if (!HasInput && llvm::sys::Process::StandardInIsUserInput()) { - //const unsigned ID = Diags->getCustomDiagID( - //clang::DiagnosticsEngine::Level::Error, - //"Problems loading PCH: '%0'."); + if (!HasInput && llvm::sys::Process::StandardInIsUserInput()) { + const unsigned ID = Diags->getCustomDiagID( + clang::DiagnosticsEngine::Level::Error, + "Problems loading PCH: '%0'."); - //Diags->Report(ID) << PCHFile; - //// If this was the only error, then don't let it stop anything - //if (Diags->getClient()->getNumErrors() == 1) - //Diags->Reset(true); - //// Clear the include so no one else uses it. - //std::string().swap(PCHFile); - //} + Diags->Report(ID) << PCHFile; + // If this was the only error, then don't let it stop anything + if (Diags->getClient()->getNumErrors() == 1) + Diags->Reset(true); + // Clear the include so no one else uses it. + std::string().swap(PCHFile); + } } // All we care about is if Language and Target options were successful. - //InitLang = !listener.m_ReadLang; - //InitTarget = !listener.m_ReadTarget; - //} + InitLang = !listener.m_ReadLang; + InitTarget = !listener.m_ReadTarget; + } } FrontendOptions& FrontendOpts = Invocation.getFrontendOpts(); diff --git a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp index 876baa7f2089d..540b962449a36 100644 --- a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp +++ b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp @@ -1452,6 +1452,11 @@ void clang::InitializePreprocessor(Preprocessor &PP, // any -include directives. for (unsigned i = 0, e = InitOpts.MacroIncludes.size(); i != e; ++i) AddImplicitIncludeMacros(Builder, InitOpts.MacroIncludes[i]); + + // Process -include-pch/-include-pth directives. + if (!InitOpts.ImplicitPCHInclude.empty()) + AddImplicitIncludePCH(Builder, PP, PCHContainerRdr, + InitOpts.ImplicitPCHInclude); // Process -include directives. for (unsigned i = 0, e = InitOpts.Includes.size(); i != e; ++i) { @@ -1459,11 +1464,6 @@ void clang::InitializePreprocessor(Preprocessor &PP, AddImplicitInclude(Builder, Path); } - //// Process -include-pch/-include-pth directives. - //if (!InitOpts.ImplicitPCHInclude.empty()) - //AddImplicitIncludePCH(Builder, PP, PCHContainerRdr, - //InitOpts.ImplicitPCHInclude); - // Instruct the preprocessor to skip the preamble. PP.setSkipMainFilePreamble(InitOpts.PrecompiledPreambleBytes.first, InitOpts.PrecompiledPreambleBytes.second); From a60c2de0ba881372116057276f9bb91ae66bb3c6 Mon Sep 17 00:00:00 2001 From: Lukas Johannes Breitwieser Date: Thu, 19 Dec 2024 20:22:54 +0100 Subject: [PATCH 3/3] Misc improvements --- core/metacling/src/TCling.cxx | 13 ++++++++++++- etc/dictpch/makepch.py | 2 +- interpreter/cling/lib/Interpreter/CIFactory.cpp | 14 +++++--------- .../IncrementalCUDADeviceCompiler.cpp | 17 +++++++++-------- .../clang/lib/Frontend/InitPreprocessor.cpp | 2 +- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/core/metacling/src/TCling.cxx b/core/metacling/src/TCling.cxx index e831914ad0426..ae700e3eeec68 100644 --- a/core/metacling/src/TCling.cxx +++ b/core/metacling/src/TCling.cxx @@ -1354,6 +1354,7 @@ TCling::TCling(const char *name, const char *title, const char* const argv[], vo std::vector clingArgsStorage; clingArgsStorage.push_back("cling4root"); + std::string interpInclude(TROOT::GetEtcDir().Data()); for (const char* const* arg = argv; *arg; ++arg) clingArgsStorage.push_back(*arg); @@ -1362,7 +1363,6 @@ TCling::TCling(const char *name, const char *title, const char* const argv[], vo ROOT::TMetaUtils::SetPathsForRelocatability(clingArgsStorage); // Add -I early so ASTReader can find the headers. - std::string interpInclude(TROOT::GetEtcDir().Data()); clingArgsStorage.push_back("-I" + interpInclude); // Add include path to etc/cling. @@ -1403,6 +1403,17 @@ TCling::TCling(const char *name, const char *title, const char* const argv[], vo #endif } + // TODO(lukas) enable only if compiled with CUDA support. + clingArgsStorage.push_back("-x"); + clingArgsStorage.push_back("cuda"); + clingArgsStorage.push_back("-isystem"); + clingArgsStorage.push_back(interpInclude + "cling/lib/clang/18/include/cuda_wrappers"); + //clingArgsStorage.push_back("--cuda-gpu-arch=sm_60"); + //clingArgsStorage.push_back("-L/usr/lib/x86_64-linux-gnu"); + //clingArgsStorage.push_back("-lcuda"); + //clingArgsStorage.push_back("-lcudart"); + //clingArgsStorage.push_back("--cuda-path=/usr/lib/cuda"); + // Process externally passed arguments if present. std::optional EnvOpt = llvm::sys::Process::GetEnv("EXTRA_CLING_ARGS"); if (EnvOpt.has_value()) { diff --git a/etc/dictpch/makepch.py b/etc/dictpch/makepch.py index e58a98b3d95e6..8233f4e2f222f 100755 --- a/etc/dictpch/makepch.py +++ b/etc/dictpch/makepch.py @@ -118,7 +118,7 @@ def makepch(): ret = subprocess.call(command.split(), env=my_env) if ret == 0: shutil.move("allDict_rdict.pch",pchFileName) - # os.unlink("allDict.cxx") + os.unlink("allDict.cxx") return ret diff --git a/interpreter/cling/lib/Interpreter/CIFactory.cpp b/interpreter/cling/lib/Interpreter/CIFactory.cpp index 16ce792178bbe..e093268bd25a9 100644 --- a/interpreter/cling/lib/Interpreter/CIFactory.cpp +++ b/interpreter/cling/lib/Interpreter/CIFactory.cpp @@ -1509,16 +1509,12 @@ namespace { bool ReadTargetOptions(const TargetOptions &TargetOpts, bool /*Complain*/, bool /*AllowCompatibleDifferences*/) override { - // latest uncomment + // TODO quick workaround for enabling CUDA. The loaded PCH has an + // empty SDKVersion, which causes a compilation error. + auto PrevSDKVersion = m_Invocation.getTargetOpts().SDKVersion; m_Invocation.getTargetOpts() = TargetOpts; - auto& to = m_Invocation.getTargetOpts(); - //to.Features.clear(); - //to.CodeModel = "default"; - //to.CodeObjectVersion = llvm::COV_5; - //to.LargeDataThreshold = 0; - to.SDKVersion = VersionTuple(12,0); - //to.HLSLEntry = ""; - //m_ReadTarget = true; + m_ReadTarget = true; + m_Invocation.getTargetOpts().SDKVersion = PrevSDKVersion; return false; } bool ReadPreprocessorOptions( diff --git a/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp b/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp index 5e709c47d870d..f24a372f6d2c9 100644 --- a/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp +++ b/interpreter/cling/lib/Interpreter/IncrementalCUDADeviceCompiler.cpp @@ -61,8 +61,6 @@ namespace cling { std::string("--cuda-gpu-arch=sm_") .append(std::to_string(m_CuArgs->smVersion)), "--cuda-device-only"}; - argv.push_back("-isystem"); - argv.push_back("/home/lukas/sft/ws/root-project/build/Debug-no-modules/etc/cling/lib/clang/18/include/cuda_wrappers"); addHeaderSearchPathFlags(argv, CI.getHeaderSearchOptsPtr()); @@ -86,8 +84,8 @@ namespace cling { std::transform(argv.begin(), argv.end(), argvChar.begin(), [&](const std::string& s) { return s.c_str(); }); - // TODO investigate where the -include-pch is added - // This is a problem for builds without modules + // TODO investigate where the -include-pch argument without a file + // is added from. This is a problem for builds without modules. for (auto& el : argvChar) { //std::cout << el << std::endl; if (el != nullptr && std::string(el) == "-include-pch") { @@ -95,10 +93,6 @@ namespace cling { break; } } - //for (auto& el : argvChar) { - //std::cout << el << std::endl; - //} - //argvChar[15] = nullptr; // argv list have to finish with a nullptr. argvChar.push_back(nullptr); @@ -219,6 +213,13 @@ namespace cling { if (e.Group == clang::frontend::IncludeDirGroup::Angled) argv.push_back("-I" + e.Path); + + // make sure the cuda_wrappers directory is added as system include + // TODO ensure that this is the correct way to do that + if (e.Group == clang::frontend::IncludeDirGroup::System) { + argv.push_back("-isystem"); + argv.push_back(e.Path); + } } } diff --git a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp index 540b962449a36..1b91c86f91398 100644 --- a/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp +++ b/interpreter/llvm-project/clang/lib/Frontend/InitPreprocessor.cpp @@ -1452,7 +1452,7 @@ void clang::InitializePreprocessor(Preprocessor &PP, // any -include directives. for (unsigned i = 0, e = InitOpts.MacroIncludes.size(); i != e; ++i) AddImplicitIncludeMacros(Builder, InitOpts.MacroIncludes[i]); - + // Process -include-pch/-include-pth directives. if (!InitOpts.ImplicitPCHInclude.empty()) AddImplicitIncludePCH(Builder, PP, PCHContainerRdr,