Add opt-in multi-line real-time build status display

misc/output_test.py

@@ -392,6 +392,57 @@ def test_status_flag(self) -> None:
                      flags="--status '<${finished}/${total}> '")
         self.assertEqual(output, '<1/1> echo a\x1b[K\ndo thing\n')
 
+    def test_multiline_flag(self) -> None:
+        long_title = 'component/subdir/very/long/job/title/that/should/keep/the/right_side/of/the_name'
+        output = run(
+            f'''rule echo
+  command = printf "done"
+  description = {long_title}
+
+build a: echo
+''',
+            flags='-m',
+            raw_output=True)
+        status_line = '[1/1] ...le/that/should/keep/the/right_side/of/the_name 0.0s\x1b[K'
+        self.assertEqual(len(status_line.replace('\x1b[K', '')), 60)
+        self.assertEqual(
+            output,
+            f'{status_line}\r\n'
+            f'\x1b[J\x1b[1A{status_line}\r\n'
+            '\x1b[Jdone\r\n')
+
+    def test_multiline_single_hidden_edge(self) -> None:
+        output = run(
+'''rule echo
+  command = sleep 1 && echo $out
+  description = echo $out
+
+build a: echo
+build b: echo
+build c: echo
+build d: echo
+build e: echo
+build f: echo
+build g: echo
+build h: echo
+build i: echo
+''',
+            flags='-j9 -m',
+            raw_output=True)
+        self.assertRegex(
+            output,
+            r'(?s)^'
+            r'\[1/9\] echo a\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[2/9\] echo b\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[3/9\] echo c\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[4/9\] echo d\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[5/9\] echo e\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[6/9\] echo f\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[7/9\] echo g\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[8/9\] echo h\.+ 0\.\ds\x1b\[K\r\n'
+            r'\[9/9\] echo i\.+ 0\.\ds\x1b\[K\r\n')
+        self.assertNotIn('... and 1 more', output)
+
     def test_status_flag_unknown_variable(self) -> None:
         'Does --status fail clearly on an unknown variable?'
         self._test_expected_error(

src/build.cc

@@ -615,6 +615,7 @@ Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log,
       explanations_(g_explaining ? new Explanations(status) : nullptr),
       scan_(state, build_log, deps_log, disk_interface,
             &config_.depfile_parser_options, explanations_.get()) {
+  status->SetStartTimeMillis(start_time_millis);
   lock_file_path_ = ".ninja_lock";
   string build_dir = state_->bindings_.LookupVariable("builddir");
   if (!build_dir.empty())
@@ -715,7 +716,7 @@ ExitStatus Builder::Build(string* err) {
     if (config_.dry_run)
       command_runner_.reset(new DryRunCommandRunner);
     else
-      command_runner_.reset(CommandRunner::factory(config_, jobserver_.get()));
+      command_runner_.reset(CommandRunner::factory(config_, status_, jobserver_.get()));
     ;
   }
 
@@ -842,6 +843,7 @@ ExitStatus Builder::Build(string* err) {
   }
 
   status_->BuildFinished();
+  status_->Report();
   return ExitSuccess;
 }
 
@@ -851,6 +853,7 @@ bool Builder::StartEdge(Edge* edge, string* err) {
     return true;
 
   int64_t start_time_millis = GetTimeMillis() - start_time_millis_;
+  edge->start_time_ = start_time_millis;
   running_edges_.insert(make_pair(edge, start_time_millis));
 
   status_->BuildEdgeStarted(edge, start_time_millis);

src/build.h

@@ -178,6 +178,7 @@ struct CommandRunner {
   /// Creates the RealCommandRunner. \arg jobserver can be nullptr if there
   /// is no jobserver pool to use.
   static CommandRunner* factory(const BuildConfig& config,
+                                Status* status,
                                 Jobserver::Client* jobserver);
 };
 
@@ -192,6 +193,7 @@ struct BuildConfig {
     VERBOSE
   };
   Verbosity verbosity = NORMAL;
+  bool multiline_console = false;
   bool dry_run = false;
   int parallelism = 1;
   bool disable_jobserver_client = false;

src/graph.h

@@ -220,6 +220,8 @@ struct Edge {
   BindingEnv* env_ = nullptr;
   size_t id_ = 0;
   int64_t critical_path_weight_ = -1;
+  int64_t start_time_ = 0;
+  int sequence_ = 0;
 
   /// A Jobserver slot instance. Invalid by default.
   Jobserver::Slot job_slot_;

src/ninja.cc

@@ -233,6 +233,7 @@ void Usage(const BuildConfig& config) {
 "options:\n"
 "  --version      print ninja version (\"%s\")\n"
 "  -v, --verbose  show all command lines while building\n"
+"  -m             enable multiline status display when supported\n"
 "  --quiet        don't show progress status, just command output\n"
 "  --status FMT   progress status format using Ninja-style $vars\n"
 "                 (e.g. --status '[$finished/$total] ')\n"
@@ -1743,7 +1744,7 @@ int ReadFlags(int* argc, char*** argv,
 
   int opt;
   while (!options->tool &&
-         (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions,
+      (opt = getopt_long(*argc, *argv, "d:f:j:k:l:mnt:vw:C:h", kLongOptions,
                             NULL)) != -1) {
     switch (opt) {
       case 'd':
@@ -1788,6 +1789,9 @@ int ReadFlags(int* argc, char*** argv,
         config->max_load_average = value;
         break;
       }
+      case 'm':
+        config->multiline_console = true;
+        break;
       case 'n':
         config->dry_run = true;
         config->disable_jobserver_client = true;

src/real_command_runner.cc

@@ -16,12 +16,15 @@
 #include "exit_status.h"
 #include "jobserver.h"
 #include "limits.h"
+#include "metrics.h"
+#include "status.h"
 #include "subprocess.h"
 
 struct RealCommandRunner : public CommandRunner {
   explicit RealCommandRunner(const BuildConfig& config,
+                             Status *status,
                              Jobserver::Client* jobserver)
-      : config_(config), jobserver_(jobserver) {}
+      : config_(config), status_(status),  jobserver_(jobserver) {}
   size_t CanRunMore() const override;
   bool StartCommand(Edge* edge) override;
   BuildResult WaitForCommand() override;
@@ -38,9 +41,11 @@ struct RealCommandRunner : public CommandRunner {
   }
 
   const BuildConfig& config_;
+  Status* status_;
   SubprocessSet subprocs_;
   Jobserver::Client* jobserver_ = nullptr;
   std::map<const Subprocess*, Edge*> subproc_to_edge_;
+  int64_t timer_ = 0;
 };
 
 std::vector<Edge*> RealCommandRunner::GetActiveEdges() {
@@ -119,6 +124,11 @@ BuildResult RealCommandRunner::WaitForCommandOrJobserverToken(
   // Wait for DoWork() to report activity
   while (work_result == SubprocessSet::WorkResult::NoWork) {
     work_result = subprocs_.DoWork();
+    const int64_t now = GetTimeMillis();
+    if (now - timer_ > 100) {
+      timer_ = now;
+      status_->Report();
+    }
   }
 
   // Address interrupts first, then subprocesses finishing, then finally
@@ -155,6 +165,7 @@ BuildResult RealCommandRunner::WaitForCommandOrJobserverToken(
 }
 
 CommandRunner* CommandRunner::factory(const BuildConfig& config,
+                                      Status* status,
                                       Jobserver::Client* jobserver) {
-  return new RealCommandRunner(config, jobserver);
+  return new RealCommandRunner(config, status, jobserver);
 }

src/status.h

@@ -25,15 +25,17 @@ struct Explanations;
 /// Abstract interface to object that tracks the status of a build:
 /// completion fraction, printing updates.
 struct Status {
+  virtual void SetStartTimeMillis(int64_t start_time_millis) = 0;
   virtual void EdgeAddedToPlan(const Edge* edge) = 0;
   virtual void EdgeRemovedFromPlan(const Edge* edge) = 0;
-  virtual void BuildEdgeStarted(const Edge* edge,
+  virtual void BuildEdgeStarted(Edge* edge,
                                 int64_t start_time_millis) = 0;
   virtual void BuildEdgeFinished(Edge* edge, int64_t start_time_millis,
                                  int64_t end_time_millis, ExitStatus exit_code,
                                  const std::string& output) = 0;
   virtual void BuildStarted() = 0;
   virtual void BuildFinished() = 0;
+  virtual void Report() = 0;
 
   /// Set the Explanations instance to use to report explanations,
   /// argument can be nullptr if no explanations need to be printed