Skip to content

Commit

Permalink
Merge branch 'pr-1140-jobserver' into kitware-staged-features
Browse files Browse the repository at this point in the history
  • Loading branch information
@bradking
bradking committed May 25, 2021
2 parents ed056bd + 2aa885c commit d88bb4c
Show file tree
Hide file tree
Showing 17 changed files with 1,562 additions and 60 deletions.
8 changes: 7 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ add_library(libninja OBJECT
src/parser.cc
src/state.cc
src/string_piece_util.cc
src/tokenpool-gnu-make.cc
src/util.cc
src/version.cc
)
Expand All @@ -104,12 +105,16 @@ if(WIN32)
src/msvc_helper-win32.cc
src/msvc_helper_main-win32.cc
src/getopt.c
src/tokenpool-gnu-make-win32.cc
)
if(MSVC)
target_sources(libninja PRIVATE src/minidump-win32.cc)
endif()
else()
target_sources(libninja PRIVATE src/subprocess-posix.cc)
target_sources(libninja PRIVATE
src/subprocess-posix.cc
src/tokenpool-gnu-make-posix.cc
)
if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX")
target_sources(libninja PRIVATE src/getopt.c)
endif()
Expand Down Expand Up @@ -182,6 +187,7 @@ if(BUILD_TESTING)
src/string_piece_util_test.cc
src/subprocess_test.cc
src/test.cc
src/tokenpool_test.cc
src/util_test.cc
)
if(WIN32)
Expand Down
7 changes: 6 additions & 1 deletion configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,11 +514,13 @@ def has_re2c():
'parser',
'state',
'string_piece_util',
'tokenpool-gnu-make',
'util',
'version']:
objs += cxx(name, variables=cxxvariables)
if platform.is_windows():
for name in ['subprocess-win32',
'tokenpool-gnu-make-win32',
'includes_normalize-win32',
'msvc_helper-win32',
'msvc_helper_main-win32']:
Expand All @@ -527,7 +529,9 @@ def has_re2c():
objs += cxx('minidump-win32', variables=cxxvariables)
objs += cc('getopt')
else:
objs += cxx('subprocess-posix')
for name in ['subprocess-posix',
'tokenpool-gnu-make-posix']:
objs += cxx(name)
if platform.is_aix():
objs += cc('getopt')
if platform.is_msvc():
Expand Down Expand Up @@ -582,6 +586,7 @@ def has_re2c():
'string_piece_util_test',
'subprocess_test',
'test',
'tokenpool_test',
'util_test']:
objs += cxx(name, variables=cxxvariables)
if platform.is_windows():
Expand Down
127 changes: 92 additions & 35 deletions src/build.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#include "graph.h"
#include "state.h"
#include "subprocess.h"
#include "tokenpool.h"
#include "util.h"

using namespace std;
Expand All @@ -50,8 +51,9 @@ struct DryRunCommandRunner : public CommandRunner {

// Overridden from CommandRunner:
virtual bool CanRunMore() const;
virtual bool AcquireToken();
virtual bool StartCommand(Edge* edge);
virtual bool WaitForCommand(Result* result);
virtual bool WaitForCommand(Result* result, bool more_ready);

private:
queue<Edge*> finished_;
Expand All @@ -61,12 +63,16 @@ bool DryRunCommandRunner::CanRunMore() const {
return true;
}

bool DryRunCommandRunner::AcquireToken() {
return true;
}

bool DryRunCommandRunner::StartCommand(Edge* edge) {
finished_.push(edge);
return true;
}

bool DryRunCommandRunner::WaitForCommand(Result* result) {
bool DryRunCommandRunner::WaitForCommand(Result* result, bool more_ready) {
if (finished_.empty())
return false;

Expand Down Expand Up @@ -379,7 +385,7 @@ void Plan::EdgeWanted(const Edge* edge) {
}

Edge* Plan::FindWork() {
if (ready_.empty())
if (!more_ready())
return NULL;
set<Edge*>::iterator e = ready_.begin();
Edge* edge = *e;
Expand Down Expand Up @@ -665,19 +671,39 @@ void Plan::Dump() const {
}

struct RealCommandRunner : public CommandRunner {
explicit RealCommandRunner(const BuildConfig& config) : config_(config) {}
virtual ~RealCommandRunner() {}
explicit RealCommandRunner(const BuildConfig& config);
virtual ~RealCommandRunner();
virtual bool CanRunMore() const;
virtual bool AcquireToken();
virtual bool StartCommand(Edge* edge);
virtual bool WaitForCommand(Result* result);
virtual bool WaitForCommand(Result* result, bool more_ready);
virtual vector<Edge*> GetActiveEdges();
virtual void Abort();

const BuildConfig& config_;
// copy of config_.max_load_average; can be modified by TokenPool setup
double max_load_average_;
SubprocessSet subprocs_;
TokenPool* tokens_;
map<const Subprocess*, Edge*> subproc_to_edge_;
};

RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
max_load_average_ = config.max_load_average;
if ((tokens_ = TokenPool::Get()) != NULL) {
if (!tokens_->Setup(config_.parallelism_from_cmdline,
config_.verbosity == BuildConfig::VERBOSE,
max_load_average_)) {
delete tokens_;
tokens_ = NULL;
}
}
}

RealCommandRunner::~RealCommandRunner() {
delete tokens_;
}

vector<Edge*> RealCommandRunner::GetActiveEdges() {
vector<Edge*> edges;
for (map<const Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin();
Expand All @@ -688,34 +714,57 @@ vector<Edge*> RealCommandRunner::GetActiveEdges() {

void RealCommandRunner::Abort() {
subprocs_.Clear();
if (tokens_)
tokens_->Clear();
}

bool RealCommandRunner::CanRunMore() const {
size_t subproc_number =
subprocs_.running_.size() + subprocs_.finished_.size();
return (int)subproc_number < config_.parallelism
&& ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f)
|| GetLoadAverage() < config_.max_load_average);
bool parallelism_limit_not_reached =
tokens_ || // ignore config_.parallelism
((int) (subprocs_.running_.size() +
subprocs_.finished_.size()) < config_.parallelism);
return parallelism_limit_not_reached
&& (subprocs_.running_.empty() ||
(max_load_average_ <= 0.0f ||
GetLoadAverage() < max_load_average_));
}

bool RealCommandRunner::AcquireToken() {
return (!tokens_ || tokens_->Acquire());
}

bool RealCommandRunner::StartCommand(Edge* edge) {
string command = edge->EvaluateCommand();
Subprocess* subproc = subprocs_.Add(command, edge->use_console());
if (!subproc)
return false;
if (tokens_)
tokens_->Reserve();
subproc_to_edge_.insert(make_pair(subproc, edge));

return true;
}

bool RealCommandRunner::WaitForCommand(Result* result) {
bool RealCommandRunner::WaitForCommand(Result* result, bool more_ready) {
Subprocess* subproc;
while ((subproc = subprocs_.NextFinished()) == NULL) {
bool interrupted = subprocs_.DoWork();
subprocs_.ResetTokenAvailable();
while (((subproc = subprocs_.NextFinished()) == NULL) &&
!subprocs_.IsTokenAvailable()) {
bool interrupted = subprocs_.DoWork(more_ready ? tokens_ : NULL);
if (interrupted)
return false;
}

// token became available
if (subproc == NULL) {
result->status = ExitTokenAvailable;
return true;
}

// command completed
if (tokens_)
tokens_->Release();

result->status = subproc->Finish();
result->output = subproc->GetOutput();

Expand Down Expand Up @@ -825,45 +874,53 @@ bool Builder::Build(string* err) {
// command runner.
// Second, we attempt to wait for / reap the next finished command.
while (plan_.more_to_do()) {
// See if we can start any more commands.
if (failures_allowed && command_runner_->CanRunMore()) {
if (Edge* edge = plan_.FindWork()) {
if (edge->GetBindingBool("generator")) {
scan_.build_log()->Close();
}
// See if we can start any more commands...
bool can_run_more =
failures_allowed &&
plan_.more_ready() &&
command_runner_->CanRunMore();

// ... but we also need a token to do that.
if (can_run_more && command_runner_->AcquireToken()) {
Edge* edge = plan_.FindWork();
if (edge->GetBindingBool("generator")) {
scan_.build_log()->Close();
}
if (!StartEdge(edge, err)) {
Cleanup();
status_->BuildFinished();
return false;
}

if (!StartEdge(edge, err)) {
if (edge->is_phony()) {
if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) {
Cleanup();
status_->BuildFinished();
return false;
}

if (edge->is_phony()) {
if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) {
Cleanup();
status_->BuildFinished();
return false;
}
} else {
++pending_commands;
}

// We made some progress; go back to the main loop.
continue;
} else {
++pending_commands;
}

// We made some progress; go back to the main loop.
continue;
}

// See if we can reap any finished commands.
if (pending_commands) {
CommandRunner::Result result;
if (!command_runner_->WaitForCommand(&result) ||
if (!command_runner_->WaitForCommand(&result, can_run_more) ||
result.status == ExitInterrupted) {
Cleanup();
status_->BuildFinished();
*err = "interrupted by user";
return false;
}

// We might be able to start another command; start the main loop over.
if (result.status == ExitTokenAvailable)
continue;

--pending_commands;
if (!FinishCommand(&result, err)) {
Cleanup();
Expand Down
12 changes: 10 additions & 2 deletions src/build.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ struct Plan {
/// Returns true if there's more work to be done.
bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; }

/// Returns true if there's more edges ready to start
bool more_ready() const { return !ready_.empty(); }

/// Dumps the current state of the plan.
void Dump() const;

Expand Down Expand Up @@ -139,6 +142,7 @@ struct Plan {
struct CommandRunner {
virtual ~CommandRunner() {}
virtual bool CanRunMore() const = 0;
virtual bool AcquireToken() = 0;
virtual bool StartCommand(Edge* edge) = 0;

/// The result of waiting for a command.
Expand All @@ -150,15 +154,18 @@ struct CommandRunner {
bool success() const { return status == ExitSuccess; }
};
/// Wait for a command to complete, or return false if interrupted.
virtual bool WaitForCommand(Result* result) = 0;
/// If more_ready is true then the optional TokenPool is monitored too
/// and we return when a token becomes available.
virtual bool WaitForCommand(Result* result, bool more_ready) = 0;

virtual std::vector<Edge*> GetActiveEdges() { return std::vector<Edge*>(); }
virtual void Abort() {}
};

/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
BuildConfig() : verbosity(NORMAL), dry_run(false),
parallelism(1), parallelism_from_cmdline(false),
failures_allowed(1), max_load_average(-0.0f) {}

enum Verbosity {
Expand All @@ -169,6 +176,7 @@ struct BuildConfig {
Verbosity verbosity;
bool dry_run;
int parallelism;
bool parallelism_from_cmdline;
int failures_allowed;
/// The maximum load average we must not exceed. A negative value
/// means that we do not have any limit.
Expand Down
Loading

0 comments on commit d88bb4c

Please sign in to comment.