Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve module deps #2237

Merged
merged 2 commits into from
Apr 3, 2022
Merged

Improve module deps #2237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd159c7
improve c++ modules and jobpool
waruqi Apr 3, 2022
69fe42c
improve module parser
waruqi Apr 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions xmake/modules/private/async/jobpool.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import("core.base.object")
import("core.base.hashset")

-- define module
local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs"}}
local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs", "_poprefs"}}

-- get jobs size
function jobpool:size()
Expand Down Expand Up @@ -128,13 +128,16 @@ function jobpool:pop()
end
end

-- is group node?
if job.group then
-- is group node or referenced node (it has been popped once) ?
local poprefs = self._poprefs
local jobkey = tostring(job)
if job.group or poprefs[jobkey] then
-- pop the next real job
return self:pop()
else
-- pop this job
self._size = self._size - 1
poprefs[jobkey] = true
return job, priority
end
end
Expand Down Expand Up @@ -214,5 +217,5 @@ end

-- new a jobpool
function new()
return jobpool {0, {name = "root"}, {}}
return jobpool {0, {name = "root"}, {}, {}}
end
22 changes: 5 additions & 17 deletions xmake/rules/c++/modules/build_modules/clang.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,26 +74,23 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
end

-- load moduledeps
local moduledeps = module_parser.load(target, sourcebatch, opt)

-- build moduledeps
local moduledeps_files = module_parser.build(moduledeps)
local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)

-- compile module files to object files
local count = 0
local modulefiles = {}
local sourcefiles_total = #sourcebatch.sourcefiles
for i = 1, sourcefiles_total do
local sourcefile = sourcebatch.sourcefiles[i]
local moduledep = moduledeps_files[sourcefile] or {}
local moduleinfo = moduledeps_files[sourcefile] or {}

-- make module file path, @note we need process submodule name, e.g. module.submodule.mpp -> module.submodule.pcm
-- @see https://github.com/xmake-io/xmake/pull/1982
local modulefile = path.join(cachedir, (moduledep.name or path.basename(sourcefile)) .. ".pcm")
local modulefile = path.join(cachedir, (moduleinfo.name or path.basename(sourcefile)) .. ".pcm")
table.insert(modulefiles, modulefile)

-- make build job
moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)

-- compile module files to *.pcm
local opt2 = table.join(opt, {configs = {force = {cxxflags = {modulesflag,
Expand Down Expand Up @@ -128,14 +125,5 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
end

-- build batchjobs
local rootjob = opt.rootjob
for _, moduledep in pairs(moduledeps) do
if moduledep.parents then
for _, parent in ipairs(moduledep.parents) do
batchjobs:add(moduledep.job, parent.job)
end
else
batchjobs:add(moduledep.job, rootjob)
end
end
module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
end
20 changes: 4 additions & 16 deletions xmake/rules/c++/modules/build_modules/gcc.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,13 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
end

-- load moduledeps
local moduledeps = module_parser.load(target, sourcebatch, opt)

-- build moduledeps
local moduledeps_files = module_parser.build(moduledeps)
local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)

-- compile module files to object files
for i = 1, #sourcebatch.sourcefiles do
local sourcefile = sourcebatch.sourcefiles[i]
local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile)
moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile)
moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
local opt2 = table.join(opt, {configs = {force = {cxxflags = {"-x c++"}}}})
opt2.progress = (index * 100) / total
opt2.objectfile = sourcebatch.objectfiles[i]
Expand All @@ -82,15 +79,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
target:add("cxxflags", modulesflag)

-- build batchjobs
local rootjob = opt.rootjob
for _, moduledep in pairs(moduledeps) do
if moduledep.parents then
for _, parent in ipairs(moduledep.parents) do
batchjobs:add(moduledep.job, parent.job)
end
else
batchjobs:add(moduledep.job, rootjob)
end
end
module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
end

59 changes: 44 additions & 15 deletions xmake/rules/c++/modules/build_modules/module_parser.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

-- imports
import("core.project.depend")
import("core.base.hashset")

-- get depend file of module source file
function _get_dependfile_of_modulesource(target, sourcefile)
Expand Down Expand Up @@ -70,6 +71,25 @@ function _generate_moduledeps(target, sourcefile, opt)
end, {dependfile = dependfile, files = {sourcefile}})
end

-- build batch jobs with deps
function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
local targetjob_ref = jobrefs[moduleinfo.name]
if targetjob_ref then
batchjobs:add(targetjob_ref, rootjob)
else
local modulejob = batchjobs:add(moduleinfo.job, rootjob)
if modulejob then
jobrefs[moduleinfo.name] = modulejob
for _, depname in ipairs(moduleinfo.deps) do
local dep = moduledeps[depname]
if dep then -- maybe nil, e.g. `import <string>;`
_build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, dep)
end
end
end
end
end

-- generate module deps
function generate(target, sourcebatch, opt)
for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
Expand Down Expand Up @@ -98,23 +118,32 @@ function load(target, sourcebatch, opt)
end
end
end
return moduledeps
end

-- build module deps
function build(moduledeps)
-- get moduledeps with file map
local moduledeps_files = {}
for _, moduledep in pairs(moduledeps) do
if moduledep.deps then
for _, depname in ipairs(moduledep.deps) do
local dep = moduledeps[depname]
if dep then
dep.parents = dep.parents or {}
table.insert(dep.parents, moduledep)
end
end
for _, moduleinfo in pairs(moduledeps) do
moduledeps_files[moduleinfo.file] = moduleinfo
end
return moduledeps, moduledeps_files
end

-- build batch jobs
function build_batchjobs(moduledeps, batchjobs, rootjob)
local depset = hashset.new()
for _, moduleinfo in pairs(moduledeps) do
assert(moduleinfo.job)
for _, depname in ipairs(moduleinfo.deps) do
depset:insert(depname)
end
end
local moduledeps_root = {}
for _, moduleinfo in pairs(moduledeps) do
if not depset:has(moduleinfo.name) then
table.insert(moduledeps_root, moduleinfo)
end
moduledeps_files[moduledep.file] = moduledep
end
return moduledeps_files
local jobrefs = {}
for _, moduleinfo in pairs(moduledeps_root) do
_build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
end
end
20 changes: 4 additions & 16 deletions xmake/rules/c++/modules/build_modules/msvc.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,18 +116,15 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
end

-- load moduledeps
local moduledeps = module_parser.load(target, sourcebatch, opt)

-- build moduledeps
local moduledeps_files = module_parser.build(moduledeps)
local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)

-- compile module files to object files
local count = 0
local sourcefiles_total = #sourcebatch.sourcefiles
for i = 1, sourcefiles_total do
local sourcefile = sourcebatch.sourcefiles[i]
local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile)
moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile)
moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
local opt2 = table.join(opt, {configs = {force = {cxxflags = {
interfaceflag,
{outputflag, modulefiles[i]},
Expand Down Expand Up @@ -169,15 +166,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
end

-- build batchjobs
local rootjob = opt.rootjob
for _, moduledep in pairs(moduledeps) do
if moduledep.parents then
for _, parent in ipairs(moduledep.parents) do
batchjobs:add(moduledep.job, parent.job)
end
else
batchjobs:add(moduledep.job, rootjob)
end
end
module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
end