From cd159c7c62fb2746f85484fa813628598d068072 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 3 Apr 2022 13:40:24 +0800 Subject: [PATCH 1/2] improve c++ modules and jobpool --- xmake/modules/private/async/jobpool.lua | 11 ++-- .../rules/c++/modules/build_modules/clang.lua | 22 ++------ xmake/rules/c++/modules/build_modules/gcc.lua | 20 ++----- .../modules/build_modules/module_parser.lua | 56 ++++++++++++++----- .../rules/c++/modules/build_modules/msvc.lua | 20 ++----- 5 files changed, 61 insertions(+), 68 deletions(-) diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua index f7eef0c1bfa..458b8802d81 100644 --- a/xmake/modules/private/async/jobpool.lua +++ b/xmake/modules/private/async/jobpool.lua @@ -23,7 +23,7 @@ import("core.base.object") import("core.base.hashset") -- define module -local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs"}} +local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs", "_poprefs"}} -- get jobs size function jobpool:size() @@ -128,13 +128,16 @@ function jobpool:pop() end end - -- is group node? - if job.group then + -- is group node or referenced node (it has been popped once) ? + local poprefs = self._poprefs + local jobkey = tostring(job) + if job.group or poprefs[jobkey] then -- pop the next real job return self:pop() else -- pop this job self._size = self._size - 1 + poprefs[jobkey] = true return job, priority end end @@ -214,5 +217,5 @@ end -- new a jobpool function new() - return jobpool {0, {name = "root"}, {}} + return jobpool {0, {name = "root"}, {}, {}} end diff --git a/xmake/rules/c++/modules/build_modules/clang.lua b/xmake/rules/c++/modules/build_modules/clang.lua index 5a7b108acf3..b62f2d26918 100644 --- a/xmake/rules/c++/modules/build_modules/clang.lua +++ b/xmake/rules/c++/modules/build_modules/clang.lua @@ -74,10 +74,7 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end -- load moduledeps - local moduledeps = module_parser.load(target, sourcebatch, opt) - - -- build moduledeps - local moduledeps_files = module_parser.build(moduledeps) + local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt) -- compile module files to object files local count = 0 @@ -85,15 +82,15 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) local sourcefiles_total = #sourcebatch.sourcefiles for i = 1, sourcefiles_total do local sourcefile = sourcebatch.sourcefiles[i] - local moduledep = moduledeps_files[sourcefile] or {} + local moduleinfo = moduledeps_files[sourcefile] or {} -- make module file path, @note we need process submodule name, e.g. module.submodule.mpp -> module.submodule.pcm -- @see https://github.com/xmake-io/xmake/pull/1982 - local modulefile = path.join(cachedir, (moduledep.name or path.basename(sourcefile)) .. ".pcm") + local modulefile = path.join(cachedir, (moduleinfo.name or path.basename(sourcefile)) .. ".pcm") table.insert(modulefiles, modulefile) -- make build job - moduledep.job = batchjobs:newjob(sourcefile, function (index, total) + moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total) -- compile module files to *.pcm local opt2 = table.join(opt, {configs = {force = {cxxflags = {modulesflag, @@ -128,14 +125,5 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end -- build batchjobs - local rootjob = opt.rootjob - for _, moduledep in pairs(moduledeps) do - if moduledep.parents then - for _, parent in ipairs(moduledep.parents) do - batchjobs:add(moduledep.job, parent.job) - end - else - batchjobs:add(moduledep.job, rootjob) - end - end + module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob) end diff --git a/xmake/rules/c++/modules/build_modules/gcc.lua b/xmake/rules/c++/modules/build_modules/gcc.lua index 2abaa4f7f99..3bf8b8fcceb 100644 --- a/xmake/rules/c++/modules/build_modules/gcc.lua +++ b/xmake/rules/c++/modules/build_modules/gcc.lua @@ -59,16 +59,13 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end -- load moduledeps - local moduledeps = module_parser.load(target, sourcebatch, opt) - - -- build moduledeps - local moduledeps_files = module_parser.build(moduledeps) + local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt) -- compile module files to object files for i = 1, #sourcebatch.sourcefiles do local sourcefile = sourcebatch.sourcefiles[i] - local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile) - moduledep.job = batchjobs:newjob(sourcefile, function (index, total) + local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile) + moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total) local opt2 = table.join(opt, {configs = {force = {cxxflags = {"-x c++"}}}}) opt2.progress = (index * 100) / total opt2.objectfile = sourcebatch.objectfiles[i] @@ -82,15 +79,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) target:add("cxxflags", modulesflag) -- build batchjobs - local rootjob = opt.rootjob - for _, moduledep in pairs(moduledeps) do - if moduledep.parents then - for _, parent in ipairs(moduledep.parents) do - batchjobs:add(moduledep.job, parent.job) - end - else - batchjobs:add(moduledep.job, rootjob) - end - end + module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob) end diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua index 5f7bee7178b..5319d5b467c 100644 --- a/xmake/rules/c++/modules/build_modules/module_parser.lua +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -20,6 +20,7 @@ -- imports import("core.project.depend") +import("core.base.hashset") -- get depend file of module source file function _get_dependfile_of_modulesource(target, sourcefile) @@ -70,6 +71,22 @@ function _generate_moduledeps(target, sourcefile, opt) end, {dependfile = dependfile, files = {sourcefile}}) end +-- build batch jobs with deps +function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo) + local targetjob_ref = jobrefs[moduleinfo.name] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local modulejob = batchjobs:add(moduleinfo.job, rootjob) + if modulejob then + jobrefs[moduleinfo.name] = modulejob + for _, depname in ipairs(moduleinfo.deps) do + _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, moduledeps[depname]) + end + end + end +end + -- generate module deps function generate(target, sourcebatch, opt) for _, sourcefile in ipairs(sourcebatch.sourcefiles) do @@ -98,23 +115,32 @@ function load(target, sourcebatch, opt) end end end - return moduledeps -end --- build module deps -function build(moduledeps) + -- get moduledeps with file map local moduledeps_files = {} - for _, moduledep in pairs(moduledeps) do - if moduledep.deps then - for _, depname in ipairs(moduledep.deps) do - local dep = moduledeps[depname] - if dep then - dep.parents = dep.parents or {} - table.insert(dep.parents, moduledep) - end - end + for _, moduleinfo in pairs(moduledeps) do + moduledeps_files[moduleinfo.file] = moduleinfo + end + return moduledeps, moduledeps_files +end + +-- build batch jobs +function build_batchjobs(moduledeps, batchjobs, rootjob) + local depset = hashset.new() + for _, moduleinfo in pairs(moduledeps) do + assert(moduleinfo.job) + for _, depname in ipairs(moduleinfo.deps) do + depset:insert(depname) end - moduledeps_files[moduledep.file] = moduledep end - return moduledeps_files + local moduledeps_root = {} + for _, moduleinfo in pairs(moduledeps) do + if not depset:has(moduleinfo.name) then + table.insert(moduledeps_root, moduleinfo) + end + end + local jobrefs = {} + for _, moduleinfo in pairs(moduledeps_root) do + _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo) + end end diff --git a/xmake/rules/c++/modules/build_modules/msvc.lua b/xmake/rules/c++/modules/build_modules/msvc.lua index cb81effd7dd..7076bcaa535 100644 --- a/xmake/rules/c++/modules/build_modules/msvc.lua +++ b/xmake/rules/c++/modules/build_modules/msvc.lua @@ -116,18 +116,15 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end -- load moduledeps - local moduledeps = module_parser.load(target, sourcebatch, opt) - - -- build moduledeps - local moduledeps_files = module_parser.build(moduledeps) + local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt) -- compile module files to object files local count = 0 local sourcefiles_total = #sourcebatch.sourcefiles for i = 1, sourcefiles_total do local sourcefile = sourcebatch.sourcefiles[i] - local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile) - moduledep.job = batchjobs:newjob(sourcefile, function (index, total) + local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile) + moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total) local opt2 = table.join(opt, {configs = {force = {cxxflags = { interfaceflag, {outputflag, modulefiles[i]}, @@ -169,15 +166,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end -- build batchjobs - local rootjob = opt.rootjob - for _, moduledep in pairs(moduledeps) do - if moduledep.parents then - for _, parent in ipairs(moduledep.parents) do - batchjobs:add(moduledep.job, parent.job) - end - else - batchjobs:add(moduledep.job, rootjob) - end - end + module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob) end From 69fe42c33e92bed0365fc3d99f0d87c88edb62ba Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 3 Apr 2022 13:51:11 +0800 Subject: [PATCH 2/2] improve module parser --- xmake/rules/c++/modules/build_modules/module_parser.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua index 5319d5b467c..91a09aadb09 100644 --- a/xmake/rules/c++/modules/build_modules/module_parser.lua +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -81,7 +81,10 @@ function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, mod if modulejob then jobrefs[moduleinfo.name] = modulejob for _, depname in ipairs(moduleinfo.deps) do - _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, moduledeps[depname]) + local dep = moduledeps[depname] + if dep then -- maybe nil, e.g. `import ;` + _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, dep) + end end end end