Jobserver fifo

src/build.cc

@@ -474,9 +474,9 @@ struct RealCommandRunner : public CommandRunner {
 RealCommandRunner::RealCommandRunner(const BuildConfig& config) : config_(config) {
   max_load_average_ = config.max_load_average;
   if ((tokens_ = TokenPool::Get()) != NULL) {
-    if (!tokens_->Setup(config_.parallelism_from_cmdline,
-                        config_.verbosity == BuildConfig::VERBOSE,
-                        max_load_average_)) {
+    if (!tokens_->SetupClient(config_.parallelism_from_cmdline,
+                              config_.verbosity == BuildConfig::VERBOSE,
+                              max_load_average_)) {
       delete tokens_;
       tokens_ = NULL;
     }

src/subprocess_test.cc

@@ -40,7 +40,8 @@ struct TestTokenPool : public TokenPool {
   void Reserve()     {}
   void Release()     {}
   void Clear()       {}
-  bool Setup(bool ignore_unused, bool verbose, double& max_load_average) { return false; }
+  bool SetupClient(bool ignore_unused, bool verbose,
+                   double& max_load_average) { return false; }
 
 #ifdef _WIN32
   bool _token_available;

src/tokenpool-gnu-make-posix.cc

@@ -40,6 +40,7 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
  private:
   int rfd_;
   int wfd_;
+  bool closeFds_;
 
   struct sigaction old_act_;
   bool restore_;
@@ -48,14 +49,19 @@ struct GNUmakeTokenPoolPosix : public GNUmakeTokenPool {
   static void CloseDupRfd(int signum);
 
   bool CheckFd(int fd);
+  bool CheckFifo(const char* fifo);
   bool SetAlarmHandler();
 };
 
-GNUmakeTokenPoolPosix::GNUmakeTokenPoolPosix() : rfd_(-1), wfd_(-1), restore_(false) {
+GNUmakeTokenPoolPosix::GNUmakeTokenPoolPosix() : rfd_(-1), wfd_(-1), closeFds_(false), restore_(false) {
 }
 
 GNUmakeTokenPoolPosix::~GNUmakeTokenPoolPosix() {
   Clear();
+  if (closeFds_) {
+    close(wfd_);
+    close(rfd_);
+  }
   if (restore_)
     sigaction(SIGALRM, &old_act_, NULL);
 }
@@ -69,6 +75,36 @@ bool GNUmakeTokenPoolPosix::CheckFd(int fd) {
   return true;
 }
 
+bool GNUmakeTokenPoolPosix::CheckFifo(const char* fifo) {
+  // remove possible junk from end of fifo name
+  char *filename = strdup(fifo);
+  char *end;
+  if ((end = strchr(filename, ' ')) != NULL) {
+    *end = '\0';
+  }
+
+  int rfd = open(filename, O_RDONLY);
+  if (rfd < 0) {
+    free(filename);
+    return false;
+  }
+
+  int wfd = open(filename, O_WRONLY);
+  if (wfd < 0) {
+    close(rfd);
+    free(filename);
+    return false;
+  }
+
+  free(filename);
+
+  rfd_ = rfd;
+  wfd_ = wfd;
+  closeFds_ = true;
+
+  return true;
+}
+
 int GNUmakeTokenPoolPosix::dup_rfd_ = -1;
 
 void GNUmakeTokenPoolPosix::CloseDupRfd(int signum) {
@@ -89,6 +125,13 @@ bool GNUmakeTokenPoolPosix::SetAlarmHandler() {
 }
 
 bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) {
+  // check for jobserver-fifo style
+  const char* fifo;
+  if (((fifo = strstr(jobserver, "=fifo:")) != NULL) &&
+      CheckFifo(fifo + 6))
+    return SetAlarmHandler();
+
+  // check for legacy simple pipe style
   int rfd = -1;
   int wfd = -1;
   if ((sscanf(jobserver, "%*[^=]=%d,%d", &rfd, &wfd) == 2) &&
@@ -100,6 +143,7 @@ bool GNUmakeTokenPoolPosix::ParseAuth(const char* jobserver) {
     return true;
   }
 
+  // some jobserver style we don't support
   return false;
 }
 

src/tokenpool-gnu-make.cc

@@ -28,9 +28,9 @@ GNUmakeTokenPool::GNUmakeTokenPool() : available_(1), used_(0) {
 GNUmakeTokenPool::~GNUmakeTokenPool() {
 }
 
-bool GNUmakeTokenPool::Setup(bool ignore,
-                             bool verbose,
-                             double& max_load_average) {
+bool GNUmakeTokenPool::SetupClient(bool ignore,
+                                   bool verbose,
+                                   double& max_load_average) {
   const char* value = GetEnv("MAKEFLAGS");
   if (!value)
     return false;

src/tokenpool-gnu-make.h

@@ -24,7 +24,8 @@ struct GNUmakeTokenPool : public TokenPool {
   virtual void Reserve();
   virtual void Release();
   virtual void Clear();
-  virtual bool Setup(bool ignore, bool verbose, double& max_load_average);
+  virtual bool SetupClient(bool ignore, bool verbose,
+                           double& max_load_average);
 
   // platform specific implementation
   virtual const char* GetEnv(const char* name) = 0;

src/tokenpool.h

@@ -26,7 +26,8 @@ struct TokenPool {
   virtual void Clear() = 0;
 
   // returns false if token pool setup failed
-  virtual bool Setup(bool ignore, bool verbose, double& max_load_average) = 0;
+  virtual bool SetupClient(bool ignore, bool verbose,
+                           double& max_load_average) = 0;
 
 #ifdef _WIN32
   virtual void WaitForTokenAvailability(HANDLE ioport) = 0;

src/tokenpool_test.cc

@@ -19,7 +19,10 @@
 #ifdef _WIN32
 #include <windows.h>
 #else
+#include <fcntl.h>
 #include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #endif
 
 #include <stdio.h>
@@ -35,6 +38,8 @@
 #define AUTH_FORMAT(tmpl)   "foo " tmpl "=%d,%d bar"
 #define ENVIRONMENT_CLEAR() unsetenv("MAKEFLAGS")
 #define ENVIRONMENT_INIT(v) setenv("MAKEFLAGS", v, true)
+
+#define FIFO_NAME "ninja-test-tokenpool-fifo"
 #endif
 
 namespace {
@@ -60,11 +65,24 @@ struct TokenPoolTest : public testing::Test {
     semaphore_name_ = SEMAPHORE_NAME;
     if ((semaphore_ = CreateSemaphore(0, 0, 2, SEMAPHORE_NAME)) == NULL)
 #else
+    if (mkfifo(FIFO_NAME, 0600) < 0) {
+      ASSERT_TRUE(false);
+    }
+
     if (pipe(fds_) < 0)
 #endif
       ASSERT_TRUE(false);
   }
 
+  void GetPool(bool ignore_jobserver) {
+    if ((tokens_ = TokenPool::Get()) != NULL) {
+      if (!tokens_->SetupClient(ignore_jobserver, false, load_avg_)) {
+        delete tokens_;
+        tokens_ = NULL;
+      }
+    }
+  }
+
   void CreatePool(const char* format, bool ignore_jobserver = false) {
     if (format) {
       sprintf(buf_, format,
@@ -76,18 +94,30 @@ struct TokenPoolTest : public testing::Test {
       );
       ENVIRONMENT_INIT(buf_);
     }
-    if ((tokens_ = TokenPool::Get()) != NULL) {
-      if (!tokens_->Setup(ignore_jobserver, false, load_avg_)) {
-        delete tokens_;
-        tokens_ = NULL;
-      }
-    }
+    GetPool(ignore_jobserver);
   }
 
   void CreateDefaultPool() {
     CreatePool(AUTH_FORMAT("--jobserver-auth"));
   }
 
+#ifndef _WIN32
+  void CreateFifoPool() {
+    // close simple pipe fds...
+    close(fds_[0]);
+    close(fds_[1]);
+
+    // ... and open named pipe instead
+    if ((fds_[0] = open(FIFO_NAME, O_RDONLY|O_NONBLOCK)) < 0)
+      ASSERT_TRUE(false);
+    if ((fds_[1] = open(FIFO_NAME, O_WRONLY)) < 0)
+      ASSERT_TRUE(false);
+
+    ENVIRONMENT_INIT("foo --jobserver-auth=fifo:" FIFO_NAME " bar");
+    GetPool(false);
+  }
+#endif
+
   virtual void TearDown() {
     if (tokens_)
       delete tokens_;
@@ -96,6 +126,7 @@ struct TokenPoolTest : public testing::Test {
 #else
     close(fds_[0]);
     close(fds_[1]);
+    unlink(FIFO_NAME);
 #endif
     ENVIRONMENT_CLEAR();
   }
@@ -167,6 +198,15 @@ TEST_F(TokenPoolTest, MonitorFD) {
 
   EXPECT_EQ(fds_[0], tokens_->GetMonitorFd());
 }
+
+TEST_F(TokenPoolTest, MonitorFDFifo) {
+  CreateFifoPool();
+
+  ASSERT_NE(NULL, tokens_);
+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+  EXPECT_NE(-1, tokens_->GetMonitorFd());
+}
 #endif
 
 TEST_F(TokenPoolTest, ImplicitToken) {
@@ -226,6 +266,42 @@ TEST_F(TokenPoolTest, TwoTokens) {
   EXPECT_TRUE(tokens_->Acquire());
 }
 
+#ifndef _WIN32
+TEST_F(TokenPoolTest, TwoTokensFifo) {
+  CreateFifoPool();
+
+  ASSERT_NE(NULL, tokens_);
+  EXPECT_EQ(kLoadAverageDefault, load_avg_);
+
+  // implicit token
+  EXPECT_TRUE(tokens_->Acquire());
+  tokens_->Reserve();
+  EXPECT_FALSE(tokens_->Acquire());
+
+  // jobserver offers 2nd token
+  char test_tokens[1] = { '+' };
+  ASSERT_EQ(1u, write(fds_[1], test_tokens, sizeof(test_tokens)));
+  EXPECT_TRUE(tokens_->Acquire());
+  tokens_->Reserve();
+  EXPECT_FALSE(tokens_->Acquire());
+
+  // release 2nd token
+  tokens_->Release();
+  EXPECT_TRUE(tokens_->Acquire());
+
+  // release implicit token - must return 2nd token back to jobserver
+  tokens_->Release();
+  EXPECT_TRUE(tokens_->Acquire());
+
+  // there must be one token available
+  EXPECT_EQ(1u, read(fds_[0], buf_, sizeof(buf_)));
+  EXPECT_EQ(test_tokens[0], buf_[0]);
+
+  // implicit token
+  EXPECT_TRUE(tokens_->Acquire());
+}
+#endif
+
 TEST_F(TokenPoolTest, Clear) {
   CreateDefaultPool();