src/subprocess-posix.cc - third_party/github.com/ninja-build/ninja - Git at Google

 // Copyright 2012 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 #ifdef __linux__
 // Ensure POSIX_SPAWN_SETSID is available on Linux
 #define _GNU_SOURCE 1
 #endif  // __linux__

 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <spawn.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/ioctl.h>
 #include <sys/select.h>
 #include <sys/wait.h>
 #include <unistd.h>

 // For openpty() which is provided by different system headers in Linux and BSDs
 // or MacOS
 #ifdef __linux__
 #include <pty.h>
 #else  // !__linux__
 #include <util.h>
 #endif

 extern char** environ;

 #include "exit_status.h"
 #include "process_tree.h"
 #include "process_utils.h"
 #include "subprocess.h"
 #include "util.h"

 #ifndef POSIX_SPAWN_SETSID
 #ifdef __linux__
 // The old Linux sysroot used by the Fuchsia recipe does not have
 // POSIX_SPAWN_SETSID
 #define POSIX_SPAWN_SETSID 0x80
 #warning "POSIX_SPAWN_SETSID defined explicitly, please update your sysroot"
 #else
 #error "POSIX_SPAWN_SETSID is not defined, please update your sysroot!"
 #endif
 #endif  // !defined(POSIX_SPAWN_SETSID)

 using namespace std;

 namespace {
 ExitStatus ParseExitStatus(int status);
 }

 Subprocess::Subprocess(SubprocessSet& subprocess_set, bool use_console)
     : subprocess_set_(subprocess_set) {
   // In order to capture console command outputs, allocate a pseudo
   // terminal if Ninja it itself running in an interactive terminal,
   // otherwise just use a pipe buffer.
   if (use_console) {
     if (isatty(1) && isatty(2)) {
       run_mode_ = RunMode::Session;
       output_mode_ = OutputMode::PseudoTerminalProxy;
     } else {
       run_mode_ = RunMode::Group;
       output_mode_ = OutputMode::PipeProxy;
     }
   } else {
     run_mode_ = RunMode::Group;
     output_mode_ = OutputMode::PipeBuffer;
   }
 }

 Subprocess::~Subprocess() {
   Stop();
   // Reap child if forgotten.
   if (!reaped_) {
     Finish();
   }
 }

 void Subprocess::Stop() {
   async_fd_.reset();
   async_timer_.reset();
 }

 bool Subprocess::Start(const string& command, const std::string* description) {
   description_ = description ? *description : command;

   ScopedHandle fd, subproc_stdout_fd;

   switch (output_mode_) {
   case OutputMode::PipeBuffer:
   case OutputMode::PipeProxy: {
     int output_pipe[2];
     if (pipe(output_pipe) < 0)
       ErrnoFatal("pipe");
     fd = output_pipe[0];
     fd.SetInheritable(false);
     subproc_stdout_fd = output_pipe[1];
     break;
   }
   case OutputMode::PseudoTerminalProxy: {
     struct winsize ws;
     if (ioctl(1, TIOCGWINSZ, &ws) < 0) {
       ws.ws_row = 24;
       ws.ws_col = 80;
       ws.ws_xpixel = 0;
       ws.ws_ypixel = 0;
     }
     struct termios tios;
     if (tcgetattr(1, &tios) < 0)
       ErrnoFatal("tcgetattr");

     // Raw mode, see 'man tcgetattr'.
     tios.c_lflag &= ~(ICANON | ISIG | IEXTEN | ECHO);
     tios.c_iflag &= ~(BRKINT | ICRNL | IGNBRK | IGNCR | INLCR | INPCK | ISTRIP |
                       IXON | PARMRK);
     tios.c_oflag &= ~(OPOST | ONLCR | OCRNL | ONOCR | ONLRET | OFILL | OFDEL);

     int pty_ninja, pty_subprocess;
     if (openpty(&pty_ninja, &pty_subprocess, NULL, &tios, &ws) < 0)
       ErrnoFatal("openpty");
     fd = pty_ninja;
     fd.SetInheritable(false);
     subproc_stdout_fd = pty_subprocess;
     break;
   }
   case OutputMode::Direct:;
   }

   posix_spawn_file_actions_t action;
   int err = posix_spawn_file_actions_init(&action);
   if (err != 0)
     ErrnoFatal("posix_spawn_file_actions_init", err);

   posix_spawnattr_t attr;
   err = posix_spawnattr_init(&attr);
   if (err != 0)
     ErrnoFatal("posix_spawnattr_init", err);

   short flags = 0;

   flags |= POSIX_SPAWN_SETSIGMASK;
   sigset_t old_mask = subprocess_set_.async_loop_.GetOldSignalMask();

 #ifndef _NDEBUG
   // Consistency check, ensure that SIGINT/SIGHUP/SIGTERM can reach
   // spawned processes.
   if (sigismember(&old_mask, SIGINT))
     Fatal("SubprocessSet: SIGINT is blocked in current signal mask");
   if (sigismember(&old_mask, SIGHUP))
     Fatal("SubprocessSet: SIGHUP is blocked in current signal mask");
   if (sigismember(&old_mask, SIGTERM))
     Fatal("SubprocessSet: SIGTERM is blocked in current signal mask");
 #endif  // !_NDEBUG

   err = posix_spawnattr_setsigmask(&attr, &old_mask);
   if (err != 0)
     ErrnoFatal("posix_spawnattr_setsigmask", err);
   // Signals which are set to be caught in the calling process image are set to
   // default action in the new process image, so no explicit
   // POSIX_SPAWN_SETSIGDEF parameter is needed.

   if (run_mode_ == RunMode::Group) {
     // Put the child in its own process group, so ctrl-c won't reach it.
     flags |= POSIX_SPAWN_SETPGROUP;
     // No need to posix_spawnattr_setpgroup(&attr, 0), it's the default.
   } else if (run_mode_ == RunMode::Session) {
     // Put the child in its own session. It will become the leader and will
     // be able to write to the pseudo terminal. Ctrl-C won't reach it too.
     flags |= POSIX_SPAWN_SETSID;
   }

 #ifdef POSIX_SPAWN_USEVFORK
   flags |= POSIX_SPAWN_USEVFORK;
 #endif

   err = posix_spawnattr_setflags(&attr, flags);
   if (err != 0)
     ErrnoFatal("posix_spawnattr_setflags", err);

   if (output_mode_ != OutputMode::Direct) {
     if (output_mode_ == OutputMode::PseudoTerminalProxy) {
       err =
           posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 0);
       if (err != 0)
         ErrnoFatal("posix_spawn_file_actions_adddup2", err);
     } else {
       // Open /dev/null over stdin.
       err = posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY,
                                              0);
       if (err != 0)
         ErrnoFatal("posix_spawn_file_actions_addopen", err);
     }

     err = posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 1);
     if (err != 0)
       ErrnoFatal("posix_spawn_file_actions_adddup2", err);
     err = posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 2);
     if (err != 0)
       ErrnoFatal("posix_spawn_file_actions_adddup2", err);
     err = posix_spawn_file_actions_addclose(&action, subproc_stdout_fd.get());
     if (err != 0)
       ErrnoFatal("posix_spawn_file_actions_addclose", err);
   }

   char** env = environ;
   if (subprocess_set_.environment_)
     env = subprocess_set_.environment_->AsExecEnvironmentBlock();

   const char* spawned_args[] = { "/bin/sh", "-c", command.c_str(), NULL };
   err = posix_spawn(&pid_, "/bin/sh", &action, &attr,
                     const_cast<char**>(spawned_args), env);
   if (err != 0)
     ErrnoFatal("posix_spawn", err);

   err = posix_spawnattr_destroy(&attr);
   if (err != 0)
     ErrnoFatal("posix_spawnattr_destroy", err);
   err = posix_spawn_file_actions_destroy(&action);
   if (err != 0)
     ErrnoFatal("posix_spawn_file_actions_destroy", err);

   if (output_mode_ != OutputMode::Direct) {
     async_fd_ = AsyncHandle::Create(
         std::move(fd), subprocess_set_.async_loop_,
         [this](AsyncError error, size_t size) {
           if (error && error != EIO)
             Fatal("read: %s", strerror(error));
           if (size == 0) {
             Stop();
             subprocess_set_.OnProcessCompletion(this);
             return;
           }
           // Append result, continue reading.
           buf_.append(read_buf_, size);
           if (output_mode_ == OutputMode::PipeProxy ||
               output_mode_ == OutputMode::PseudoTerminalProxy) {
             fwrite(read_buf_, size, 1, stdout);
             fflush(stdout);
           }
           async_fd_->StartRead(read_buf_, sizeof(read_buf_));
           if (async_timer_) {
             // Add 30 (by default) more seconds to timer expiration after bytes
             async_timer_->SetDurationMs(subprocess_set_.pipe_timeout_ms_);
           }
         });
     async_fd_->StartRead(read_buf_, sizeof(read_buf_));
   }

   return true;
 }

 bool Subprocess::TryFinish(int waitpid_options) {
   assert(!reaped_ && pid_ != -1);
   int status, ret;
   while ((ret = waitpid(pid_, &status, waitpid_options)) < 0) {
     if (errno != EINTR)
       Fatal("waitpid(%d): %s", pid_, strerror(errno));
   }
   if (ret == 0)
     return false;  // Subprocess is alive (WNOHANG-only).
   reaped_ = true;
   exit_status_ = ParseExitStatus(status);
   return true;  // Subprocess has terminated.
 }

 ExitStatus Subprocess::Finish() {
   if (!reaped_) {
     TryFinish(0);
     assert(reaped_);
   }
   return exit_status_;
 }

 namespace {

 ExitStatus ParseExitStatus(int status) {
 #ifdef _AIX
   if (WIFEXITED(status) && WEXITSTATUS(status) & 0x80) {
     // Map the shell's exit code used for signal failure (128 + signal) to the
     // status code expected by AIX WIFSIGNALED and WTERMSIG macros which, unlike
     // other systems, uses a different bit layout.
     int signal = WEXITSTATUS(status) & 0x7f;
     status = (signal << 16) | signal;
   }
 #endif

   if (WIFEXITED(status)) {
     // propagate the status transparently
     return static_cast<ExitStatus>(WEXITSTATUS(status));
   } else if (WIFSIGNALED(status)) {
     if (WTERMSIG(status) == SIGINT || WTERMSIG(status) == SIGTERM
         || WTERMSIG(status) == SIGHUP)
       return ExitInterrupted;
   }

   // At this point, we exit with any other signal+128
   return static_cast<ExitStatus>(status + 128);
 }

 }  // anonymous namespace

 bool Subprocess::Done() const {
   if (run_mode_ == RunMode::Local)
     return reaped_;

   return !async_fd_;
 }

 const string& Subprocess::GetOutput() const {
   return buf_;
 }

 SubprocessSet::SubprocessSet() : SubprocessSet(AsyncLoop::Get(), nullptr) {}

 SubprocessSet::SubprocessSet(AsyncLoop& async_loop,
                              const EnvironmentBlock* environment)
     : async_loop_(async_loop), interrupt_catcher_(async_loop_),
       environment_(environment),
       async_sigchild_(AsyncSigChild::Create(
           async_loop_, [this]() { this->CheckForTerminatedProcesses(); })) {}

 SubprocessSet::~SubprocessSet() {
   Clear();
 }

 // Reaps processes that have exited and moves them from the running set to the
 // finished set.
 void SubprocessSet::CheckForTerminatedProcesses() {
   for (auto i = running_.begin(); i != running_.end();) {
     auto& subproc = *i;
     if (subproc->reaped_ || !subproc->TryFinish(WNOHANG)) {
       // This process is either still running, or already finished.
       ++i;
       continue;
     }

     if (!subproc->async_fd_) {
       // This process is reaped and its pipe is no longer open, it can be
       // moved to the finished set now.
       finished_.push(std::move(subproc));
       i = running_.erase(i);
       continue;
     }

     // The process is reaped but its pipe is still open. Setup a timer to wait
     // for incoming bytes in the pipe. After 30 seconds (by default) the
     // subprocess will be force-finished when it fires.
     //
     // This is important to catch cases where the subprocess launched a
     // daemonized or background process that inherits the pipe descriptor
     // and keeps it open. Normally, such a process should close its
     // stdout/stderr descriptors but this is not always the case.
     subproc->async_timer_ = AsyncTimer::CreateWithDuration(
         pipe_timeout_ms_, async_loop_, [this, subproc = subproc.get()]() {
           // NOTE: Do not call subproc->Done() which will deletei the timer.
           // from this callback, leading to crashes.
           subproc->async_fd_.reset();
           subproc->force_finished_ = true;
           this->OnProcessCompletion(subproc);
         });
     ++i;
   }
 }

 Subprocess* SubprocessSet::Add(const string& command,
                                const std::string* description,
                                bool use_console) {
   auto subprocess = std::make_unique<Subprocess>(*this, use_console);
   if (!subprocess->Start(command, description))
     return nullptr;

   Subprocess* result = subprocess.get();
   running_.push_back(std::move(subprocess));
   return result;
 }

 void SubprocessSet::OnProcessCompletion(Subprocess* subprocess) {
   // Move the subprocess from the running_ vector to the finished_ queue.
   for (auto it = running_.begin(); it != running_.end(); ++it) {
     if (it->get() == subprocess) {
       finished_.emplace(it->release());
       running_.erase(it);
       return;
     }
   }
 }

 void SubprocessSet::SetJobserverFD(int fd) {
   if (!async_jobserver_ready_) {
     async_jobserver_ready_ = AsyncFdReadyFlags::CreateForRead(async_loop_, fd);
   } else {
     async_jobserver_ready_->ResetFd(fd);
   }
 }

 SubprocessSet::WorkResult SubprocessSet::DoWork() {
   size_t running_count = running_.size();
   if (!running_count)
     return WorkResult::NoWork;

   do {
     AsyncLoop::ExitStatus status = async_loop_.RunOnce(-1);

     // First detect user interruption.
     if (status == AsyncLoop::ExitInterrupted) {
       return WorkResult::Interrupted;
     }

     // Check for process completion.
     CheckForTerminatedProcesses();
     if (running_.size() != running_count) {
       return WorkResult::SubprocFinished;
     }

     // Finally, check for jobserver token availability, if needed.
     if (async_jobserver_ready_ && async_jobserver_ready_->IsReadReady())
       return WorkResult::JobserverTokenAvailable;
   } while (true);
 }

 std::unique_ptr<Subprocess> SubprocessSet::NextFinished() {
   if (finished_.empty())
     return {};

   std::unique_ptr<Subprocess> subproc = std::move(finished_.front());
   finished_.pop();
   return subproc;
 }

 void SubprocessSet::ReportRunningSubprocesses(const ProcessTree::SystemSnapshot& snapshot) {
   int num_commands = static_cast<int>(running_.size());
   if (num_commands == 0)
     return;

   // Take a system-wide snapshot of all process hierarchies exactly once.
   // This avoids redundant system calls (like scanning /proc) in the loop below.

   const int max_commands = 8;
   int nn = 0;
   for (auto& subproc : running_) {
     // Only report up to a certain number of commands to avoid flooding stderr.
     if (++nn > max_commands) {
       fprintf(stderr, "  ... and %d more\n", num_commands - max_commands);
       break;
     }

     fprintf(stderr, "- Waiting for %s[PID %6d]: %s\n",
             subproc->reaped_ ? "REAPED " : "", subproc->pid_,
             subproc->description_.c_str());

     // Build a focused tree containing only the descendants of THIS subprocess
     // using the pre-captured system-wide snapshot.
     ProcessTree tree(subproc->pid_, snapshot);

     // Attempt to gather best-effort details (like command lines) for this
     // specific subtree.
     tree.CollectDetails();

     // Use Ninja's internal description for the root of the tree, as it's
     // more accurate than what the OS might provide (e.g. it knows about
     // target names).
     std::unordered_map<ProcessTree::Pid, std::string> known_descriptions;
     known_descriptions[subproc->pid_] = subproc->description_;

     std::string output;
     // Format and append the subtree representation to 'output'.
     tree.Print(1, known_descriptions, &output);
     if (!output.empty())
       fprintf(stderr, "%s", output.c_str());
   }
 }

 void SubprocessSet::Clear() {
   int interrupted = async_loop_.GetInterruptSignal();

   // Capture the process tree snapshot BEFORE killing subprocesses.
   // Otherwise, they might exit before we can scan /proc.
   ProcessTree::SystemSnapshot snapshot;
   if (interrupted && !running_.empty()) {
     snapshot = ProcessTree::TakeSystemSnapshot();
   }
   if (interrupted) {
     async_loop_.ClearInterrupt();

     for (auto& subproc : running_) {
       // Since console processes are in our process group, they will receive
       // the interruption signal (i.e. SIGINT or SIGTERM) at the same time as
       // us.
       if (!subproc->reaped_ && subproc->run_mode_ != Subprocess::RunMode::Local)
         kill(-subproc->pid_, interrupted);
     }
   }

   // If there are reaped processes with pending pipe timers, force-finish them
   // because they will never be reaped again.
   auto force_finish_reaped_processes = [&]() {
     for (auto it = running_.begin(); it != running_.end();) {
       Subprocess* subproc = it->get();
       if (subproc->reaped_ && subproc->async_timer_) {
         subproc->Stop();
         subproc->force_finished_ = true;
         subproc->exit_status_ = ExitInterrupted;
         finished_.emplace(it->release());
         it = running_.erase(it);
       } else {
         ++it;
       }
     }
   };

   // Wait for all running processes to complete cleanly.
   // If the user presses Ctrl-C once, print a message to the user.
   // If the user presses Ctrl-C twice, force the exit.
   int interrupt_count = 0;

   // Respond with the process tree dump immediately on the first SIGTERM.
   // This helps identify what's running when Ninja is terminated gracefully
   // by an external tool.
   if (interrupted == SIGTERM && running_.size() > 0) {
     fprintf(stderr,
             "\n\nNinja received SIGTERM. Waiting for %d command%s to complete "
             "properly.\n\n",
             (int)running_.size(), running_.size() == 1 ? "" : "s");
     ReportRunningSubprocesses(snapshot);
     interrupt_count = 1;
   }

   force_finish_reaped_processes();

   while (running_.size()) {
     AsyncLoop::ExitStatus loop_status = async_loop_.RunOnce(-1);
     if (loop_status == AsyncLoop::ExitInterrupted) {
       async_loop_.ClearInterrupt();
       int num_commands = static_cast<int>(running_.size());
       if (++interrupt_count == 1) {
         // First manual interrupt (Ctrl-C) after the build was already stopped.
         // Report and polite shutdown request.
         fprintf(stderr,
                 "\n\nNinja is still waiting for %d command%s to complete "
                 "properly. Sending SIGTERM\n"
                 "to politely ask them to stop immediately. If this does not "
                 "work, press Ctrl-C\n"
                 "again to forcefully kill them. This may leave your system in "
                 "inconsistent state\n"
                 "and is not recommended.\n\n",
                 num_commands, num_commands == 1 ? "" : "s");

         ReportRunningSubprocesses(snapshot);

         force_finish_reaped_processes();

         for (auto& subproc : running_) {
           if (!subproc->reaped_)
             killpg(subproc->pid_, SIGTERM);
         }
       } else {
         // User pressed Ctrl-C twice after seeing the warning message. To force
         // the exit, first close the read end of each subprocess' output pipe,
         // print a message to the user, then send SIGKILL to the process.
         fprintf(stderr,
                 "\nKilling %d remaining command%s as requested by user.\n\n",
                 num_commands, num_commands == 1 ? "" : "s");

         force_finish_reaped_processes();

         for (auto& subproc : running_) {
           fprintf(stderr, "- Killing [PID %6d]: %s\n", subproc->pid_,
                   subproc->description_.c_str());

           // kill the process, it will be reaped in the Subprocess destructor.
           killpg(subproc->pid_, SIGKILL);
           subproc->Stop();
           subproc->reaped_ = true;
           subproc->force_finished_ = true;
           subproc->exit_status_ = ExitInterrupted;
         }
         fprintf(stderr, "\n");
         break;
       }
     }
     CheckForTerminatedProcesses();
   };

   running_.clear();
 }
	// Copyright 2012 Google Inc. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	#ifdef __linux__
	// Ensure POSIX_SPAWN_SETSID is available on Linux
	#define _GNU_SOURCE 1
	#endif // __linux__

	#include <assert.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <spawn.h>
	#include <stdio.h>
	#include <string.h>
	#include <sys/ioctl.h>
	#include <sys/select.h>
	#include <sys/wait.h>
	#include <unistd.h>

	// For openpty() which is provided by different system headers in Linux and BSDs
	// or MacOS
	#ifdef __linux__
	#include <pty.h>
	#else // !__linux__
	#include <util.h>
	#endif

	extern char** environ;

	#include "exit_status.h"
	#include "process_tree.h"
	#include "process_utils.h"
	#include "subprocess.h"
	#include "util.h"

	#ifndef POSIX_SPAWN_SETSID
	#ifdef __linux__
	// The old Linux sysroot used by the Fuchsia recipe does not have
	// POSIX_SPAWN_SETSID
	#define POSIX_SPAWN_SETSID 0x80
	#warning "POSIX_SPAWN_SETSID defined explicitly, please update your sysroot"
	#else
	#error "POSIX_SPAWN_SETSID is not defined, please update your sysroot!"
	#endif
	#endif // !defined(POSIX_SPAWN_SETSID)

	using namespace std;

	namespace {
	ExitStatus ParseExitStatus(int status);
	}

	Subprocess::Subprocess(SubprocessSet& subprocess_set, bool use_console)
	: subprocess_set_(subprocess_set) {
	// In order to capture console command outputs, allocate a pseudo
	// terminal if Ninja it itself running in an interactive terminal,
	// otherwise just use a pipe buffer.
	if (use_console) {
	if (isatty(1) && isatty(2)) {
	run_mode_ = RunMode::Session;
	output_mode_ = OutputMode::PseudoTerminalProxy;
	} else {
	run_mode_ = RunMode::Group;
	output_mode_ = OutputMode::PipeProxy;
	}
	} else {
	run_mode_ = RunMode::Group;
	output_mode_ = OutputMode::PipeBuffer;
	}
	}

	Subprocess::~Subprocess() {
	Stop();
	// Reap child if forgotten.
	if (!reaped_) {
	Finish();
	}
	}

	void Subprocess::Stop() {
	async_fd_.reset();
	async_timer_.reset();
	}

	bool Subprocess::Start(const string& command, const std::string* description) {
	description_ = description ? *description : command;

	ScopedHandle fd, subproc_stdout_fd;

	switch (output_mode_) {
	case OutputMode::PipeBuffer:
	case OutputMode::PipeProxy: {
	int output_pipe[2];
	if (pipe(output_pipe) < 0)
	ErrnoFatal("pipe");
	fd = output_pipe[0];
	fd.SetInheritable(false);
	subproc_stdout_fd = output_pipe[1];
	break;
	}
	case OutputMode::PseudoTerminalProxy: {
	struct winsize ws;
	if (ioctl(1, TIOCGWINSZ, &ws) < 0) {
	ws.ws_row = 24;
	ws.ws_col = 80;
	ws.ws_xpixel = 0;
	ws.ws_ypixel = 0;
	}
	struct termios tios;
	if (tcgetattr(1, &tios) < 0)
	ErrnoFatal("tcgetattr");

	// Raw mode, see 'man tcgetattr'.
	tios.c_lflag &= ~(ICANON \| ISIG \| IEXTEN \| ECHO);
	tios.c_iflag &= ~(BRKINT \| ICRNL \| IGNBRK \| IGNCR \| INLCR \| INPCK \| ISTRIP \|
	IXON \| PARMRK);
	tios.c_oflag &= ~(OPOST \| ONLCR \| OCRNL \| ONOCR \| ONLRET \| OFILL \| OFDEL);

	int pty_ninja, pty_subprocess;
	if (openpty(&pty_ninja, &pty_subprocess, NULL, &tios, &ws) < 0)
	ErrnoFatal("openpty");
	fd = pty_ninja;
	fd.SetInheritable(false);
	subproc_stdout_fd = pty_subprocess;
	break;
	}
	case OutputMode::Direct:;
	}

	posix_spawn_file_actions_t action;
	int err = posix_spawn_file_actions_init(&action);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_init", err);

	posix_spawnattr_t attr;
	err = posix_spawnattr_init(&attr);
	if (err != 0)
	ErrnoFatal("posix_spawnattr_init", err);

	short flags = 0;

	flags \|= POSIX_SPAWN_SETSIGMASK;
	sigset_t old_mask = subprocess_set_.async_loop_.GetOldSignalMask();

	#ifndef _NDEBUG
	// Consistency check, ensure that SIGINT/SIGHUP/SIGTERM can reach
	// spawned processes.
	if (sigismember(&old_mask, SIGINT))
	Fatal("SubprocessSet: SIGINT is blocked in current signal mask");
	if (sigismember(&old_mask, SIGHUP))
	Fatal("SubprocessSet: SIGHUP is blocked in current signal mask");
	if (sigismember(&old_mask, SIGTERM))
	Fatal("SubprocessSet: SIGTERM is blocked in current signal mask");
	#endif // !_NDEBUG

	err = posix_spawnattr_setsigmask(&attr, &old_mask);
	if (err != 0)
	ErrnoFatal("posix_spawnattr_setsigmask", err);
	// Signals which are set to be caught in the calling process image are set to
	// default action in the new process image, so no explicit
	// POSIX_SPAWN_SETSIGDEF parameter is needed.

	if (run_mode_ == RunMode::Group) {
	// Put the child in its own process group, so ctrl-c won't reach it.
	flags \|= POSIX_SPAWN_SETPGROUP;
	// No need to posix_spawnattr_setpgroup(&attr, 0), it's the default.
	} else if (run_mode_ == RunMode::Session) {
	// Put the child in its own session. It will become the leader and will
	// be able to write to the pseudo terminal. Ctrl-C won't reach it too.
	flags \|= POSIX_SPAWN_SETSID;
	}

	#ifdef POSIX_SPAWN_USEVFORK
	flags \|= POSIX_SPAWN_USEVFORK;
	#endif

	err = posix_spawnattr_setflags(&attr, flags);
	if (err != 0)
	ErrnoFatal("posix_spawnattr_setflags", err);

	if (output_mode_ != OutputMode::Direct) {
	if (output_mode_ == OutputMode::PseudoTerminalProxy) {
	err =
	posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 0);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_adddup2", err);
	} else {
	// Open /dev/null over stdin.
	err = posix_spawn_file_actions_addopen(&action, 0, "/dev/null", O_RDONLY,
	0);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_addopen", err);
	}

	err = posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 1);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_adddup2", err);
	err = posix_spawn_file_actions_adddup2(&action, subproc_stdout_fd.get(), 2);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_adddup2", err);
	err = posix_spawn_file_actions_addclose(&action, subproc_stdout_fd.get());
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_addclose", err);
	}

	char** env = environ;
	if (subprocess_set_.environment_)
	env = subprocess_set_.environment_->AsExecEnvironmentBlock();

	const char* spawned_args[] = { "/bin/sh", "-c", command.c_str(), NULL };
	err = posix_spawn(&pid_, "/bin/sh", &action, &attr,
	const_cast<char**>(spawned_args), env);
	if (err != 0)
	ErrnoFatal("posix_spawn", err);

	err = posix_spawnattr_destroy(&attr);
	if (err != 0)
	ErrnoFatal("posix_spawnattr_destroy", err);
	err = posix_spawn_file_actions_destroy(&action);
	if (err != 0)
	ErrnoFatal("posix_spawn_file_actions_destroy", err);

	if (output_mode_ != OutputMode::Direct) {
	async_fd_ = AsyncHandle::Create(
	std::move(fd), subprocess_set_.async_loop_,
	[this](AsyncError error, size_t size) {
	if (error && error != EIO)
	Fatal("read: %s", strerror(error));
	if (size == 0) {
	Stop();
	subprocess_set_.OnProcessCompletion(this);
	return;
	}
	// Append result, continue reading.
	buf_.append(read_buf_, size);
	if (output_mode_ == OutputMode::PipeProxy \|\|
	output_mode_ == OutputMode::PseudoTerminalProxy) {
	fwrite(read_buf_, size, 1, stdout);
	fflush(stdout);
	}
	async_fd_->StartRead(read_buf_, sizeof(read_buf_));
	if (async_timer_) {
	// Add 30 (by default) more seconds to timer expiration after bytes
	async_timer_->SetDurationMs(subprocess_set_.pipe_timeout_ms_);
	}
	});
	async_fd_->StartRead(read_buf_, sizeof(read_buf_));
	}

	return true;
	}

	bool Subprocess::TryFinish(int waitpid_options) {
	assert(!reaped_ && pid_ != -1);
	int status, ret;
	while ((ret = waitpid(pid_, &status, waitpid_options)) < 0) {
	if (errno != EINTR)
	Fatal("waitpid(%d): %s", pid_, strerror(errno));
	}
	if (ret == 0)
	return false; // Subprocess is alive (WNOHANG-only).
	reaped_ = true;
	exit_status_ = ParseExitStatus(status);
	return true; // Subprocess has terminated.
	}

	ExitStatus Subprocess::Finish() {
	if (!reaped_) {
	TryFinish(0);
	assert(reaped_);
	}
	return exit_status_;
	}

	namespace {

	ExitStatus ParseExitStatus(int status) {
	#ifdef _AIX
	if (WIFEXITED(status) && WEXITSTATUS(status) & 0x80) {
	// Map the shell's exit code used for signal failure (128 + signal) to the
	// status code expected by AIX WIFSIGNALED and WTERMSIG macros which, unlike
	// other systems, uses a different bit layout.
	int signal = WEXITSTATUS(status) & 0x7f;
	status = (signal << 16) \| signal;
	}
	#endif

	if (WIFEXITED(status)) {
	// propagate the status transparently
	return static_cast<ExitStatus>(WEXITSTATUS(status));
	} else if (WIFSIGNALED(status)) {
	if (WTERMSIG(status) == SIGINT \|\| WTERMSIG(status) == SIGTERM
	\|\| WTERMSIG(status) == SIGHUP)
	return ExitInterrupted;
	}

	// At this point, we exit with any other signal+128
	return static_cast<ExitStatus>(status + 128);
	}

	} // anonymous namespace

	bool Subprocess::Done() const {
	if (run_mode_ == RunMode::Local)
	return reaped_;

	return !async_fd_;
	}

	const string& Subprocess::GetOutput() const {
	return buf_;
	}

	SubprocessSet::SubprocessSet() : SubprocessSet(AsyncLoop::Get(), nullptr) {}

	SubprocessSet::SubprocessSet(AsyncLoop& async_loop,
	const EnvironmentBlock* environment)
	: async_loop_(async_loop), interrupt_catcher_(async_loop_),
	environment_(environment),
	async_sigchild_(AsyncSigChild::Create(
	async_loop_, [this]() { this->CheckForTerminatedProcesses(); })) {}

	SubprocessSet::~SubprocessSet() {
	Clear();
	}

	// Reaps processes that have exited and moves them from the running set to the
	// finished set.
	void SubprocessSet::CheckForTerminatedProcesses() {
	for (auto i = running_.begin(); i != running_.end();) {
	auto& subproc = *i;
	if (subproc->reaped_ \|\| !subproc->TryFinish(WNOHANG)) {
	// This process is either still running, or already finished.
	++i;
	continue;
	}

	if (!subproc->async_fd_) {
	// This process is reaped and its pipe is no longer open, it can be
	// moved to the finished set now.
	finished_.push(std::move(subproc));
	i = running_.erase(i);
	continue;
	}

	// The process is reaped but its pipe is still open. Setup a timer to wait
	// for incoming bytes in the pipe. After 30 seconds (by default) the
	// subprocess will be force-finished when it fires.
	//
	// This is important to catch cases where the subprocess launched a
	// daemonized or background process that inherits the pipe descriptor
	// and keeps it open. Normally, such a process should close its
	// stdout/stderr descriptors but this is not always the case.
	subproc->async_timer_ = AsyncTimer::CreateWithDuration(
	pipe_timeout_ms_, async_loop_, [this, subproc = subproc.get()]() {
	// NOTE: Do not call subproc->Done() which will deletei the timer.
	// from this callback, leading to crashes.
	subproc->async_fd_.reset();
	subproc->force_finished_ = true;
	this->OnProcessCompletion(subproc);
	});
	++i;
	}
	}

	Subprocess* SubprocessSet::Add(const string& command,
	const std::string* description,
	bool use_console) {
	auto subprocess = std::make_unique<Subprocess>(*this, use_console);
	if (!subprocess->Start(command, description))
	return nullptr;

	Subprocess* result = subprocess.get();
	running_.push_back(std::move(subprocess));
	return result;
	}

	void SubprocessSet::OnProcessCompletion(Subprocess* subprocess) {
	// Move the subprocess from the running_ vector to the finished_ queue.
	for (auto it = running_.begin(); it != running_.end(); ++it) {
	if (it->get() == subprocess) {
	finished_.emplace(it->release());
	running_.erase(it);
	return;
	}
	}
	}

	void SubprocessSet::SetJobserverFD(int fd) {
	if (!async_jobserver_ready_) {
	async_jobserver_ready_ = AsyncFdReadyFlags::CreateForRead(async_loop_, fd);
	} else {
	async_jobserver_ready_->ResetFd(fd);
	}
	}

	SubprocessSet::WorkResult SubprocessSet::DoWork() {
	size_t running_count = running_.size();
	if (!running_count)
	return WorkResult::NoWork;

	do {
	AsyncLoop::ExitStatus status = async_loop_.RunOnce(-1);

	// First detect user interruption.
	if (status == AsyncLoop::ExitInterrupted) {
	return WorkResult::Interrupted;
	}

	// Check for process completion.
	CheckForTerminatedProcesses();
	if (running_.size() != running_count) {
	return WorkResult::SubprocFinished;
	}

	// Finally, check for jobserver token availability, if needed.
	if (async_jobserver_ready_ && async_jobserver_ready_->IsReadReady())
	return WorkResult::JobserverTokenAvailable;
	} while (true);
	}

	std::unique_ptr<Subprocess> SubprocessSet::NextFinished() {
	if (finished_.empty())
	return {};

	std::unique_ptr<Subprocess> subproc = std::move(finished_.front());
	finished_.pop();
	return subproc;
	}

	void SubprocessSet::ReportRunningSubprocesses(const ProcessTree::SystemSnapshot& snapshot) {
	int num_commands = static_cast<int>(running_.size());
	if (num_commands == 0)
	return;

	// Take a system-wide snapshot of all process hierarchies exactly once.
	// This avoids redundant system calls (like scanning /proc) in the loop below.

	const int max_commands = 8;
	int nn = 0;
	for (auto& subproc : running_) {
	// Only report up to a certain number of commands to avoid flooding stderr.
	if (++nn > max_commands) {
	fprintf(stderr, " ... and %d more\n", num_commands - max_commands);
	break;
	}

	fprintf(stderr, "- Waiting for %s[PID %6d]: %s\n",
	subproc->reaped_ ? "REAPED " : "", subproc->pid_,
	subproc->description_.c_str());

	// Build a focused tree containing only the descendants of THIS subprocess
	// using the pre-captured system-wide snapshot.
	ProcessTree tree(subproc->pid_, snapshot);

	// Attempt to gather best-effort details (like command lines) for this
	// specific subtree.
	tree.CollectDetails();

	// Use Ninja's internal description for the root of the tree, as it's
	// more accurate than what the OS might provide (e.g. it knows about
	// target names).
	std::unordered_map<ProcessTree::Pid, std::string> known_descriptions;
	known_descriptions[subproc->pid_] = subproc->description_;

	std::string output;
	// Format and append the subtree representation to 'output'.
	tree.Print(1, known_descriptions, &output);
	if (!output.empty())
	fprintf(stderr, "%s", output.c_str());
	}
	}

	void SubprocessSet::Clear() {
	int interrupted = async_loop_.GetInterruptSignal();

	// Capture the process tree snapshot BEFORE killing subprocesses.
	// Otherwise, they might exit before we can scan /proc.
	ProcessTree::SystemSnapshot snapshot;
	if (interrupted && !running_.empty()) {
	snapshot = ProcessTree::TakeSystemSnapshot();
	}
	if (interrupted) {
	async_loop_.ClearInterrupt();

	for (auto& subproc : running_) {
	// Since console processes are in our process group, they will receive
	// the interruption signal (i.e. SIGINT or SIGTERM) at the same time as
	// us.
	if (!subproc->reaped_ && subproc->run_mode_ != Subprocess::RunMode::Local)
	kill(-subproc->pid_, interrupted);
	}
	}

	// If there are reaped processes with pending pipe timers, force-finish them
	// because they will never be reaped again.
	auto force_finish_reaped_processes = [&]() {
	for (auto it = running_.begin(); it != running_.end();) {
	Subprocess* subproc = it->get();
	if (subproc->reaped_ && subproc->async_timer_) {
	subproc->Stop();
	subproc->force_finished_ = true;
	subproc->exit_status_ = ExitInterrupted;
	finished_.emplace(it->release());
	it = running_.erase(it);
	} else {
	++it;
	}
	}
	};

	// Wait for all running processes to complete cleanly.
	// If the user presses Ctrl-C once, print a message to the user.
	// If the user presses Ctrl-C twice, force the exit.
	int interrupt_count = 0;

	// Respond with the process tree dump immediately on the first SIGTERM.
	// This helps identify what's running when Ninja is terminated gracefully
	// by an external tool.
	if (interrupted == SIGTERM && running_.size() > 0) {
	fprintf(stderr,
	"\n\nNinja received SIGTERM. Waiting for %d command%s to complete "
	"properly.\n\n",
	(int)running_.size(), running_.size() == 1 ? "" : "s");
	ReportRunningSubprocesses(snapshot);
	interrupt_count = 1;
	}

	force_finish_reaped_processes();

	while (running_.size()) {
	AsyncLoop::ExitStatus loop_status = async_loop_.RunOnce(-1);
	if (loop_status == AsyncLoop::ExitInterrupted) {
	async_loop_.ClearInterrupt();
	int num_commands = static_cast<int>(running_.size());
	if (++interrupt_count == 1) {
	// First manual interrupt (Ctrl-C) after the build was already stopped.
	// Report and polite shutdown request.
	fprintf(stderr,
	"\n\nNinja is still waiting for %d command%s to complete "
	"properly. Sending SIGTERM\n"
	"to politely ask them to stop immediately. If this does not "
	"work, press Ctrl-C\n"
	"again to forcefully kill them. This may leave your system in "
	"inconsistent state\n"
	"and is not recommended.\n\n",
	num_commands, num_commands == 1 ? "" : "s");

	ReportRunningSubprocesses(snapshot);

	force_finish_reaped_processes();

	for (auto& subproc : running_) {
	if (!subproc->reaped_)
	killpg(subproc->pid_, SIGTERM);
	}
	} else {
	// User pressed Ctrl-C twice after seeing the warning message. To force
	// the exit, first close the read end of each subprocess' output pipe,
	// print a message to the user, then send SIGKILL to the process.
	fprintf(stderr,
	"\nKilling %d remaining command%s as requested by user.\n\n",
	num_commands, num_commands == 1 ? "" : "s");

	force_finish_reaped_processes();

	for (auto& subproc : running_) {
	fprintf(stderr, "- Killing [PID %6d]: %s\n", subproc->pid_,
	subproc->description_.c_str());

	// kill the process, it will be reaped in the Subprocess destructor.
	killpg(subproc->pid_, SIGKILL);
	subproc->Stop();
	subproc->reaped_ = true;
	subproc->force_finished_ = true;
	subproc->exit_status_ = ExitInterrupted;
	}
	fprintf(stderr, "\n");
	break;
	}
	}
	CheckForTerminatedProcesses();
	};

	running_.clear();
	}