diff --git a/configure.py b/configure.py index 1cfa87af9d..e13111f044 100755 --- a/configure.py +++ b/configure.py @@ -272,6 +272,8 @@ def _run_command(self, cmdline): def src(filename): return os.path.join('$root', 'src', filename) +def frontend(filename): + return os.path.join('$root', 'frontend', filename) def built(filename): return os.path.join('$builddir', filename) def doc(filename): @@ -475,6 +477,43 @@ def has_re2c(): "changes to src/*.in.cc will not affect your build.") n.newline() +n.comment('the proto descriptor is generated using protoc.') +def has_protoc(): + try: + proc = subprocess.Popen(['protoc', '--version'], stdout=subprocess.PIPE) + return proc.communicate()[0].startswith("libprotoc") + except OSError: + return False + +def can_generate_proto_header(): + try: + tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py') + proc = subprocess.Popen([tool, '--probe'], stdout=subprocess.PIPE) + return proc.communicate()[0].startswith("ok") + except OSError: + return False + +if has_protoc() and can_generate_proto_header(): + # Use protoc to write out frontend.proto converted to a descriptor proto + n.rule('protoc', + command='protoc $in -o $out', + description='PROTOC $out') + n.build(frontend('frontend.pb'), 'protoc', src('frontend.proto')) + + # Use generate_proto_header.py to read in the descriptor proto and write + # a header containing field numbers and types. + n.rule('generate_proto_header', + command='$tool $in $out', + description='GEN $out') + # Generate the .h file in the source directory so we can check them in. + tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py') + n.build(src('frontend.pb.h'), 'generate_proto_header', frontend('frontend.pb'), + implicit=[tool], variables=[('tool', tool)]) +else: + print("warning: A version of protoc or the python protobuf library was not found; " + "changes to src/frontend.proto will not affect your build.") +n.newline() + n.comment('Core source files all build into ninja library.') for name in ['build', 'build_log', @@ -498,7 +537,7 @@ def has_re2c(): 'string_piece_util', 'util', 'version']: - objs += cxx(name) + objs += cxx(name, order_only=src('frontend.pb.h')) if platform.is_windows(): for name in ['subprocess-win32', 'includes_normalize-win32', diff --git a/frontend/FRONTEND.md b/frontend/FRONTEND.md new file mode 100644 index 0000000000..676ffc30b9 --- /dev/null +++ b/frontend/FRONTEND.md @@ -0,0 +1,54 @@ +Ninja Frontend Interface +======================== + +Ninja can use another program as a frontend to display build status information. +This document describes the interface between Ninja and the frontend. + +Connecting +---------- + +The frontend is passed to Ninja using a --frontend argument. The argument is +executed the same as a build rule Ninja, wrapped in `sh -c` on Linux. The +frontend will be executed with the read end of a pipe open on file descriptor +`3`. + +Ninja will pass [Protocol Buffers](https://developers.google.com/protocol-buffers/) generated from src/frontend.proto. + +stdin/stdout/stderr +------------------- + +The frontend will have stdin, stdout, and stderr connected to the same file +descriptors as Ninja. The frontend MAY read from stdin, however, if it does, +it MUST NOT read from stdin whenever a job in the console pool is running, +from when an `EdgeStarted` message is received with the `use_console` value +set to `true`, to when an `EdgeFinished` message is received with the same value +for `id`. Console rules may write directly to the same stdout/stderr as the +frontend. + +Exiting +------- + +The frontend MUST exit when the input pipe on fd `3` is closed. When a build +finishes, either successfully, due to error, or on interrupt, Ninja will close +the pipe and then block until the frontend exits. + +Experimenting with frontends +---------------------------- + +To run Ninja with a frontend that mimics the behavior of Ninja's normal output: +``` +$ ./ninja --frontend=frontend/native.py +``` + +To save serialized output to a file: +``` +$ ./ninja --frontend='cat /proc/self/fd/3 > ninja.pb all +``` + +To run a frontend with serialized input from a file: +``` +$ frontend/native.py 3< ninja.pb +``` + +The serialized output of a clean Ninja build is included in `frontend/ninja.pb`. + diff --git a/frontend/dump.py b/frontend/dump.py new file mode 100755 index 0000000000..d276fd06a3 --- /dev/null +++ b/frontend/dump.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import sys + +import frontend + +def main(): + if len(sys.argv) >= 2: + f = open(sys.argv[1], 'rb') + else: + f = frontend.default_reader() + + for msg in frontend.Frontend(f): + print('---------------------------------') + sys.stdout.write(str(msg)) + +if __name__ == '__main__': + main() + diff --git a/frontend/frontend.pb b/frontend/frontend.pb new file mode 100644 index 0000000000..d8024e33c2 Binary files /dev/null and b/frontend/frontend.pb differ diff --git a/frontend/frontend.py b/frontend/frontend.py new file mode 100755 index 0000000000..ac8fa258d4 --- /dev/null +++ b/frontend/frontend.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +"""Ninja frontend interface. + +This module implements a Ninja frontend interface that delegates handling each +message to a handler object +""" + +import os +import sys + +import google.protobuf.descriptor_pb2 +import google.protobuf.message_factory + +def default_reader(): + fd = 3 + return os.fdopen(fd, 'rb', 0) + +class Frontend(object): + """Generator class that parses length-delimited ninja status messages + through a ninja frontend interface. + """ + + def __init__(self, reader=default_reader()): + self.reader = reader + self.status_class = self.get_status_proto() + + def get_status_proto(self): + fd_set = google.protobuf.descriptor_pb2.FileDescriptorSet() + descriptor = os.path.join(os.path.dirname(__file__), 'frontend.pb') + with open(descriptor, 'rb') as f: + fd_set.ParseFromString(f.read()) + + if len(fd_set.file) != 1: + raise RuntimeError('expected exactly one file descriptor in ' + descriptor) + + messages = google.protobuf.message_factory.GetMessages(fd_set.file) + return messages['ninja.Status'] + + def __iter__(self): + return self + + def __next__(self): + return self.next() + + def next(self): + size = 0 + shift = 0 + while True: + byte = bytearray(self.reader.read(1)) + if len(byte) == 0: + raise StopIteration() + + byte = byte[0] + size += (byte & 0x7f) << (shift * 7) + if (byte & 0x80) == 0: + break + shift += 1 + if shift > 4: + raise RuntimeError('Expected varint32 length-delimeted message') + + message = self.reader.read(size) + if len(message) != size: + raise EOFError('Unexpected EOF reading %d bytes' % size) + + return self.status_class.FromString(message) diff --git a/frontend/native.py b/frontend/native.py new file mode 100755 index 0000000000..3794707ade --- /dev/null +++ b/frontend/native.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import collections +import ctypes +import os +import re +import struct +import sys + +import frontend + +class SlidingRateInfo(object): + def __init__(self, n=32): + self.rate = -1 + self.last_update = -1 + self.times = collections.deque(maxlen=n) + + def update_rate(self, update_hint, time_millis): + if update_hint == self.last_update: + return + + self.last_update = update_hint + + if len(self.times) == self.times.maxlen: + self.times.popleft() + self.times.append(time_millis) + if self.times[-1] != self.times[0]: + self.rate = len(self.times) / ((self.times[-1] - self.times[0]) / 1e3) + +strip_ansi_re = re.compile(r'\x1B\[[^a-zA-Z]*[a-zA-Z]') +def strip_ansi_escape_codes(output): + return strip_ansi_re.sub('', output) + +class NinjaNativeFrontend: + def __init__(self): + self.total_edges = 0 + self.running_edges = 0 + self.started_edges = 0 + self.finished_edges = 0 + self.running = {} + + self.time_millis = 0 + + self.progress_status_format = os.getenv('NINJA_STATUS', '[%f/%t] ') + self.current_rate = SlidingRateInfo() + self.console_locked = False + + self.printer = LinePrinter() + self.verbose = False + + def handle(self, msg): + handled = False + if msg.HasField('total_edges'): + handled = True + self.total_edges = msg.total_edges.total_edges + + if msg.HasField('build_started'): + handled = True + self.verbose = msg.build_started.verbose + self.current_rate = SlidingRateInfo(msg.build_started.parallelism) + self.running_edges = 0 + self.started_edges = 0 + self.finished_edges = 0 + self.running = {} + + if msg.HasField('build_finished'): + handled = True + self.printer.set_console_locked(False) + self.printer.print_on_new_line('') + + if msg.HasField('edge_started'): + handled = True + self.started_edges += 1 + self.running_edges += 1 + self.running[msg.edge_started.id] = msg.edge_started + self.time_millis = msg.edge_started.start_time + if msg.edge_started.console or self.printer.smart_terminal: + self.print_status(msg.edge_started) + if msg.edge_started.console: + self.printer.set_console_locked(True) + + if msg.HasField('edge_finished'): + handled = True + self.finished_edges += 1 + self.time_millis = msg.edge_finished.end_time + + edge_started = self.running[msg.edge_finished.id] + + if edge_started.console: + self.printer.set_console_locked(False) + + if not edge_started.console: + self.print_status(edge_started) + + self.running_edges -= 1 + del self.running[msg.edge_finished.id] + + # Print the command that is spewing before printing its output. + if msg.edge_finished.status != 0: + self.printer.print_on_new_line('FAILED: ' + ' '.join(edge_started.outputs)) + self.printer.print_on_new_line(edge_started.command) + + # ninja sets stdout and stderr of subprocesses to a pipe, to be able to + # check if the output is empty. Some compilers, e.g. clang, check + # isatty(stderr) to decide if they should print colored output. + # To make it possible to use colored output with ninja, subprocesses should + # be run with a flag that forces them to always print color escape codes. + # To make sure these escape codes don't show up in a file if ninja's output + # is piped to a file, ninja strips ansi escape codes again if it's not + # writing to a |smart_terminal_|. + # (Launching subprocesses in pseudo ttys doesn't work because there are + # only a few hundred available on some systems, and ninja can launch + # thousands of parallel compile commands.) + # TODO: There should be a flag to disable escape code stripping. + if msg.edge_finished.output != '': + if not self.printer.smart_terminal: + msg.edge_finished.output = strip_ansi_escape_codes(msg.edge_finished.output) + self.printer.print_on_new_line(msg.edge_finished.output) + + if msg.HasField('message'): + handled = True + # TODO(colincross): get the enum values from proto + if msg.message.level == 0: + prefix = 'ninja: ' + elif msg.message.level == 1: + prefix = 'ninja: warning: ' + elif msg.message.level == 2: + prefix = 'ninja: error: ' + self.printer.print_line(prefix + msg.message.message, LinePrinter.LINE_FULL) + + if not handled: + pass + + + def format_progress_status(self, fmt): + out = '' + fmt_iter = iter(fmt) + for c in fmt_iter: + if c == '%': + c = next(fmt_iter) + if c == '%': + out += c + elif c == 's': + out += str(self.started_edges) + elif c == 't': + out += str(self.total_edges) + elif c == 'r': + out += str(self.running_edges) + elif c == 'u': + out += str(self.total_edges - self.started_edges) + elif c == 'f': + out += str(self.finished_edges) + elif c == 'o': + if self.time_millis > 0: + rate = self.finished_edges / (self.time_millis / 1e3) + out += '{:.1f}'.format(rate) + else: + out += '?' + elif c == 'c': + self.current_rate.update_rate(self.finished_edges, self.time_millis) + if self.current_rate.rate == -1: + out += '?' + else: + out += '{:.1f}'.format(self.current_rate.rate) + elif c == 'p': + out += '{:3d}%'.format((100 * self.finished_edges) // self.total_edges) + elif c == 'e': + out += '{:.3f}'.format(self.time_millis / 1e3) + else: + raise RuntimeError('unknown placeholder '' + c +'' in $NINJA_STATUS') + else: + out += c + return out + + def print_status(self, edge_started): + to_print = edge_started.desc + if self.verbose or to_print == '': + to_print = edge_started.command + + to_print = self.format_progress_status(self.progress_status_format) + to_print + + self.printer.print_line(to_print, LinePrinter.LINE_FULL if self.verbose else LinePrinter.LINE_ELIDE) + + +def elide_middle(status, width): + margin = 3 # Space for '...'. + if len(status) + margin > width: + elide_size = (width - margin) // 2 + status = status[0:elide_size] + '...' + status[-elide_size:] + return status + +class LinePrinter(object): + LINE_FULL = 1 + LINE_ELIDE = 2 + + def __init__(self): + # Whether we can do fancy terminal control codes. + self.smart_terminal = False + + # Whether the caret is at the beginning of a blank line. + self.have_blank_line = True + + # Whether console is locked. + self.console_locked = False + + # Buffered current line while console is locked. + self.line_buffer = '' + + # Buffered line type while console is locked. + self.line_type = self.LINE_FULL + + # Buffered console output while console is locked. + self.output_buffer = '' + + if os.name == 'windows': + STD_OUTPUT_HANDLE = -11 + self.console = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) + csbi = ctypes.create_string_buffer(22) + self.smart_terminal = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi) + else: + term = os.getenv('TERM') + self.smart_terminal = os.isatty(1) and term != '' and term != 'dumb' + + def print_line(self, to_print, line_type): + if self.console_locked: + self.line_buffer = to_print + self.line_type = line_type + + if self.smart_terminal: + sys.stdout.write('\r') # Print over previous line, if any. + + if self.smart_terminal and line_type == self.LINE_ELIDE: + if os.name == 'windows': + csbi = ctypes.create_string_buffer(22) + ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi) + (cols, rows) = struct.unpack('hh', csbi.raw) + to_print = elide_middle(to_print, cols) + # TODO: windows support + # We don't want to have the cursor spamming back and forth, so instead of + # printf use WriteConsoleOutput which updates the contents of the buffer, + # but doesn't move the cursor position. + sys.stdout.write(to_print) + sys.stdout.flush() + else: + # Limit output to width of the terminal if provided so we don't cause + # line-wrapping. + import fcntl, termios + winsize = fcntl.ioctl(0, termios.TIOCGWINSZ, '\0'*4) + (rows, cols) = struct.unpack('hh', winsize) + to_print = elide_middle(to_print, cols) + sys.stdout.write(to_print) + sys.stdout.write('\x1B[K') # Clear to end of line. + sys.stdout.flush() + + self.have_blank_line = False + else: + sys.stdout.write(to_print + '\n') + sys.stdout.flush() + + def print_or_buffer(self, to_print): + if self.console_locked: + self.output_buffer += to_print + else: + sys.stdout.write(to_print) + sys.stdout.flush() + + def print_on_new_line(self, to_print): + if self.console_locked or self.line_buffer != '': + self.output_buffer += self.line_buffer + '\n' + self.line_buffer = '' + if not self.have_blank_line: + self.print_or_buffer('\n') + if to_print != '': + self.print_or_buffer(to_print) + self.have_blank_line = to_print == '' or to_print[0] == '\n' + + def set_console_locked(self, locked): + if locked == self.console_locked: + return + + if locked: + self.print_on_new_line('\n') + + self.console_locked = locked + + if not locked: + self.print_on_new_line(self.output_buffer) + if self.line_buffer != '': + self.print_line(self.line_buffer, self.line_type) + self.output_buffer = '' + self.line_buffer = '' + +def main(): + native = NinjaNativeFrontend() + for msg in frontend.Frontend(): + native.handle(msg) + +if __name__ == '__main__': + main() diff --git a/src/build.cc b/src/build.cc index 63b8f3102e..b9c16fa5e8 100644 --- a/src/build.cc +++ b/src/build.cc @@ -605,8 +605,7 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { end_time_millis = GetTimeMillis() - start_time_millis_; running_edges_.erase(i); - status_->BuildEdgeFinished(edge, end_time_millis, result->success(), - result->output); + status_->BuildEdgeFinished(edge, end_time_millis, result); // The rest of this function only applies to successful commands. if (!result->success()) { diff --git a/src/build.h b/src/build.h index 22af4a3989..5930fab48a 100644 --- a/src/build.h +++ b/src/build.h @@ -124,7 +124,8 @@ struct CommandRunner { /// Options (e.g. verbosity, parallelism) passed to a build. struct BuildConfig { BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1), - failures_allowed(1), max_load_average(-0.0f) {} + failures_allowed(1), max_load_average(-0.0f), + frontend(NULL) {} enum Verbosity { NORMAL, @@ -138,6 +139,9 @@ struct BuildConfig { /// The maximum load average we must not exceed. A negative value /// means that we do not have any limit. double max_load_average; + + /// Command to execute to handle build output + const char* frontend; }; /// Builder wraps the build process: starting commands, updating status. diff --git a/src/filebuf.h b/src/filebuf.h new file mode 100644 index 0000000000..335c5948b7 --- /dev/null +++ b/src/filebuf.h @@ -0,0 +1,49 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_FILEBUF_H_ +#define NINJA_FILEBUF_H_ + +#include + +#include + +// A non-buffering std::streambuf implementation that allows using +// a FILE* as an ostream. +class ofilebuf : public std::streambuf { + public: + ofilebuf(FILE* f) : f_(f) { } + ~ofilebuf() { } + + private: + int_type overflow(int_type c) { + if (c != traits_type::eof()) { + int ret = fputc(c, f_); + if (ret == EOF) { + return traits_type::eof(); + } + } + + return c; + } + + std::streamsize xsputn(const char* s, std::streamsize count) { + return fwrite(s, 1, count, f_); + } + + private: + FILE* f_; +}; + +#endif // NINJA_FILEBUF_H_ diff --git a/src/frontend.pb.h b/src/frontend.pb.h new file mode 100644 index 0000000000..4cd146399c --- /dev/null +++ b/src/frontend.pb.h @@ -0,0 +1,520 @@ +// This file is autogenerated by generate_proto_header.py, do not edit + +#ifndef NINJA_FRONTEND_PB_H +#define NINJA_FRONTEND_PB_H + +#include + +#include +#include +#include + +#include "proto.h" + +namespace ninja { +struct Status { + struct TotalEdges { + uint32_t total_edges_; + bool has_total_edges_; + + TotalEdges() { + total_edges_ = static_cast< uint32_t >(0); + } + + TotalEdges(const TotalEdges&); + void operator=(const TotalEdges&); + + void SerializeToOstream(std::ostream* output__) const { + WriteVarint32(output__, 1, total_edges_); + } + + size_t ByteSizeLong() const { + size_t size = 0; + size += VarintSize32(total_edges_) + 1; + return size; + } + + void Clear() { + total_edges_ = static_cast< uint32_t >(0); + } + + uint32_t* mutable_total_edges() { + has_total_edges_ = true; + return &total_edges_; + } + void set_total_edges(const uint32_t& value) { + has_total_edges_ = true; + total_edges_ = value; + } + }; + + struct BuildStarted { + uint32_t parallelism_; + bool has_parallelism_; + bool verbose_; + bool has_verbose_; + + BuildStarted() { + parallelism_ = static_cast< uint32_t >(0); + verbose_ = static_cast< bool >(0); + } + + BuildStarted(const BuildStarted&); + void operator=(const BuildStarted&); + + void SerializeToOstream(std::ostream* output__) const { + WriteVarint32(output__, 1, parallelism_); + WriteVarint32(output__, 2, verbose_); + } + + size_t ByteSizeLong() const { + size_t size = 0; + size += VarintSize32(parallelism_) + 1; + size += VarintSizeBool(verbose_) + 1; + return size; + } + + void Clear() { + parallelism_ = static_cast< uint32_t >(0); + verbose_ = static_cast< bool >(0); + } + + uint32_t* mutable_parallelism() { + has_parallelism_ = true; + return ¶llelism_; + } + void set_parallelism(const uint32_t& value) { + has_parallelism_ = true; + parallelism_ = value; + } + bool* mutable_verbose() { + has_verbose_ = true; + return &verbose_; + } + void set_verbose(const bool& value) { + has_verbose_ = true; + verbose_ = value; + } + }; + + struct BuildFinished { + BuildFinished() { + } + + BuildFinished(const BuildFinished&); + void operator=(const BuildFinished&); + + void SerializeToOstream(std::ostream* output__) const { + } + + size_t ByteSizeLong() const { + size_t size = 0; + return size; + } + + void Clear() { + } + + }; + + struct EdgeStarted { + uint32_t id_; + bool has_id_; + uint32_t start_time_; + bool has_start_time_; + std::vector< std::string > inputs_; + bool has_inputs_; + std::vector< std::string > outputs_; + bool has_outputs_; + std::string desc_; + bool has_desc_; + std::string command_; + bool has_command_; + bool console_; + bool has_console_; + + EdgeStarted() { + id_ = static_cast< uint32_t >(0); + start_time_ = static_cast< uint32_t >(0); + console_ = static_cast< bool >(0); + } + + EdgeStarted(const EdgeStarted&); + void operator=(const EdgeStarted&); + + void SerializeToOstream(std::ostream* output__) const { + WriteVarint32(output__, 1, id_); + WriteVarint32(output__, 2, start_time_); + for (std::vector< std::string >::const_iterator it_ = inputs_.begin(); + it_ != inputs_.end(); it_++) { + WriteString(output__, 3, *it_); + } + for (std::vector< std::string >::const_iterator it_ = outputs_.begin(); + it_ != outputs_.end(); it_++) { + WriteString(output__, 4, *it_); + } + WriteString(output__, 5, desc_); + WriteString(output__, 6, command_); + WriteVarint32(output__, 7, console_); + } + + size_t ByteSizeLong() const { + size_t size = 0; + size += VarintSize32(id_) + 1; + size += VarintSize32(start_time_) + 1; + for (std::vector< std::string >::const_iterator it_ = inputs_.begin(); + it_ != inputs_.end(); it_++) { + size += StringSize(*it_) + 1; + } + for (std::vector< std::string >::const_iterator it_ = outputs_.begin(); + it_ != outputs_.end(); it_++) { + size += StringSize(*it_) + 1; + } + size += StringSize(desc_) + 1; + size += StringSize(command_) + 1; + size += VarintSizeBool(console_) + 1; + return size; + } + + void Clear() { + id_ = static_cast< uint32_t >(0); + start_time_ = static_cast< uint32_t >(0); + inputs_.clear(); + outputs_.clear(); + desc_.clear(); + command_.clear(); + console_ = static_cast< bool >(0); + } + + uint32_t* mutable_id() { + has_id_ = true; + return &id_; + } + void set_id(const uint32_t& value) { + has_id_ = true; + id_ = value; + } + uint32_t* mutable_start_time() { + has_start_time_ = true; + return &start_time_; + } + void set_start_time(const uint32_t& value) { + has_start_time_ = true; + start_time_ = value; + } + std::vector< std::string >* mutable_inputs() { + has_inputs_ = true; + return &inputs_; + } + void add_inputs(const std::string& value) { + has_inputs_ = true; + inputs_.push_back(value); + } + void set_inputs(const std::vector< std::string >& value) { + has_inputs_ = true; + inputs_ = value; + } + std::vector< std::string >* mutable_outputs() { + has_outputs_ = true; + return &outputs_; + } + void add_outputs(const std::string& value) { + has_outputs_ = true; + outputs_.push_back(value); + } + void set_outputs(const std::vector< std::string >& value) { + has_outputs_ = true; + outputs_ = value; + } + std::string* mutable_desc() { + has_desc_ = true; + return &desc_; + } + void set_desc(const std::string& value) { + has_desc_ = true; + desc_ = value; + } + std::string* mutable_command() { + has_command_ = true; + return &command_; + } + void set_command(const std::string& value) { + has_command_ = true; + command_ = value; + } + bool* mutable_console() { + has_console_ = true; + return &console_; + } + void set_console(const bool& value) { + has_console_ = true; + console_ = value; + } + }; + + struct EdgeFinished { + uint32_t id_; + bool has_id_; + uint32_t end_time_; + bool has_end_time_; + int32_t status_; + bool has_status_; + std::string output_; + bool has_output_; + + EdgeFinished() { + id_ = static_cast< uint32_t >(0); + end_time_ = static_cast< uint32_t >(0); + status_ = static_cast< int32_t >(0); + } + + EdgeFinished(const EdgeFinished&); + void operator=(const EdgeFinished&); + + void SerializeToOstream(std::ostream* output__) const { + WriteVarint32(output__, 1, id_); + WriteVarint32(output__, 2, end_time_); + WriteVarint32(output__, 3, ZigZagEncode32(status_)); + WriteString(output__, 4, output_); + } + + size_t ByteSizeLong() const { + size_t size = 0; + size += VarintSize32(id_) + 1; + size += VarintSize32(end_time_) + 1; + size += VarintSize32(ZigZagEncode32(status_)) + 1; + size += StringSize(output_) + 1; + return size; + } + + void Clear() { + id_ = static_cast< uint32_t >(0); + end_time_ = static_cast< uint32_t >(0); + status_ = static_cast< int32_t >(0); + output_.clear(); + } + + uint32_t* mutable_id() { + has_id_ = true; + return &id_; + } + void set_id(const uint32_t& value) { + has_id_ = true; + id_ = value; + } + uint32_t* mutable_end_time() { + has_end_time_ = true; + return &end_time_; + } + void set_end_time(const uint32_t& value) { + has_end_time_ = true; + end_time_ = value; + } + int32_t* mutable_status() { + has_status_ = true; + return &status_; + } + void set_status(const int32_t& value) { + has_status_ = true; + status_ = value; + } + std::string* mutable_output() { + has_output_ = true; + return &output_; + } + void set_output(const std::string& value) { + has_output_ = true; + output_ = value; + } + }; + + struct Message { + enum Level { + INFO = 0, + WARNING = 1, + ERROR = 2, + }; + + ::ninja::Status::Message::Level level_; + bool has_level_; + std::string message_; + bool has_message_; + + Message() { + level_ = static_cast< ::ninja::Status::Message::Level >(0); + } + + Message(const Message&); + void operator=(const Message&); + + void SerializeToOstream(std::ostream* output__) const { + WriteVarint32SignExtended(output__, 1, static_cast(level_)); + WriteString(output__, 2, message_); + } + + size_t ByteSizeLong() const { + size_t size = 0; + size += VarintSize32SignExtended(static_cast(level_)) + 1; + size += StringSize(message_) + 1; + return size; + } + + void Clear() { + level_ = static_cast< ::ninja::Status::Message::Level >(0); + message_.clear(); + } + + ::ninja::Status::Message::Level* mutable_level() { + has_level_ = true; + return &level_; + } + void set_level(const ::ninja::Status::Message::Level& value) { + has_level_ = true; + level_ = value; + } + std::string* mutable_message() { + has_message_ = true; + return &message_; + } + void set_message(const std::string& value) { + has_message_ = true; + message_ = value; + } + }; + + ::ninja::Status::TotalEdges total_edges_; + bool has_total_edges_; + ::ninja::Status::BuildStarted build_started_; + bool has_build_started_; + ::ninja::Status::BuildFinished build_finished_; + bool has_build_finished_; + ::ninja::Status::EdgeStarted edge_started_; + bool has_edge_started_; + ::ninja::Status::EdgeFinished edge_finished_; + bool has_edge_finished_; + ::ninja::Status::Message message_; + bool has_message_; + + Status() { + } + + Status(const Status&); + void operator=(const Status&); + + void SerializeToOstream(std::ostream* output__) const { + if (has_total_edges_) { + WriteLengthDelimited(output__, 1, + total_edges_.ByteSizeLong()); + total_edges_.SerializeToOstream(output__); + } + if (has_build_started_) { + WriteLengthDelimited(output__, 2, + build_started_.ByteSizeLong()); + build_started_.SerializeToOstream(output__); + } + if (has_build_finished_) { + WriteLengthDelimited(output__, 3, + build_finished_.ByteSizeLong()); + build_finished_.SerializeToOstream(output__); + } + if (has_edge_started_) { + WriteLengthDelimited(output__, 4, + edge_started_.ByteSizeLong()); + edge_started_.SerializeToOstream(output__); + } + if (has_edge_finished_) { + WriteLengthDelimited(output__, 5, + edge_finished_.ByteSizeLong()); + edge_finished_.SerializeToOstream(output__); + } + if (has_message_) { + WriteLengthDelimited(output__, 6, + message_.ByteSizeLong()); + message_.SerializeToOstream(output__); + } + } + + size_t ByteSizeLong() const { + size_t size = 0; + if (has_total_edges_) { + size += 1 + VarintSize32(total_edges_.ByteSizeLong()); + size += total_edges_.ByteSizeLong(); + } + if (has_build_started_) { + size += 1 + VarintSize32(build_started_.ByteSizeLong()); + size += build_started_.ByteSizeLong(); + } + if (has_build_finished_) { + size += 1 + VarintSize32(build_finished_.ByteSizeLong()); + size += build_finished_.ByteSizeLong(); + } + if (has_edge_started_) { + size += 1 + VarintSize32(edge_started_.ByteSizeLong()); + size += edge_started_.ByteSizeLong(); + } + if (has_edge_finished_) { + size += 1 + VarintSize32(edge_finished_.ByteSizeLong()); + size += edge_finished_.ByteSizeLong(); + } + if (has_message_) { + size += 1 + VarintSize32(message_.ByteSizeLong()); + size += message_.ByteSizeLong(); + } + return size; + } + + void Clear() { + if (has_total_edges_) { + total_edges_.Clear(); + has_total_edges_ = false; + } + if (has_build_started_) { + build_started_.Clear(); + has_build_started_ = false; + } + if (has_build_finished_) { + build_finished_.Clear(); + has_build_finished_ = false; + } + if (has_edge_started_) { + edge_started_.Clear(); + has_edge_started_ = false; + } + if (has_edge_finished_) { + edge_finished_.Clear(); + has_edge_finished_ = false; + } + if (has_message_) { + message_.Clear(); + has_message_ = false; + } + } + + ::ninja::Status::TotalEdges* mutable_total_edges() { + has_total_edges_ = true; + return &total_edges_; + } + ::ninja::Status::BuildStarted* mutable_build_started() { + has_build_started_ = true; + return &build_started_; + } + ::ninja::Status::BuildFinished* mutable_build_finished() { + has_build_finished_ = true; + return &build_finished_; + } + ::ninja::Status::EdgeStarted* mutable_edge_started() { + has_edge_started_ = true; + return &edge_started_; + } + ::ninja::Status::EdgeFinished* mutable_edge_finished() { + has_edge_finished_ = true; + return &edge_finished_; + } + ::ninja::Status::Message* mutable_message() { + has_message_ = true; + return &message_; + } +}; + +} +#endif // NINJA_FRONTEND_PB_H diff --git a/src/frontend.proto b/src/frontend.proto new file mode 100644 index 0000000000..516745d371 --- /dev/null +++ b/src/frontend.proto @@ -0,0 +1,83 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package ninja; + +message Status { + message TotalEdges { + // New value for total edges in the build. + optional uint32 total_edges = 1; + } + + message BuildStarted { + // Number of jobs Ninja will run in parallel. + optional uint32 parallelism = 1; + // Verbose value passed to ninja. + optional bool verbose = 2; + } + + message BuildFinished { + } + + message EdgeStarted { + // Edge identification number, unique to a Ninja run. + optional uint32 id = 1; + // Edge start time in milliseconds since Ninja started. + optional uint32 start_time = 2; + // List of edge inputs. + repeated string inputs = 3; + // List of edge outputs. + repeated string outputs = 4; + // Description field from the edge. + optional string desc = 5; + // Command field from the edge. + optional string command = 6; + // Edge uses console. + optional bool console = 7; + } + + message EdgeFinished { + // Edge identification number, unique to a Ninja run. + optional uint32 id = 1; + // Edge end time in milliseconds since Ninja started. + optional uint32 end_time = 2; + // Exit status (0 for success). + optional sint32 status = 3; + // Edge output, may contain ANSI codes. + optional string output = 4; + } + + message Message { + enum Level { + INFO = 0; + WARNING = 1; + ERROR = 2; + } + // Message priority level (INFO, WARNING, or ERROR). + optional Level level = 1 [default = INFO]; + // Info/warning/error message from Ninja. + optional string message = 2; + } + + optional TotalEdges total_edges = 1; + optional BuildStarted build_started = 2; + optional BuildFinished build_finished = 3; + optional EdgeStarted edge_started = 4; + optional EdgeFinished edge_finished = 5; + optional Message message = 6; +} diff --git a/src/ninja.cc b/src/ninja.cc index c38eb3100f..7ccb1096f7 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -216,8 +216,12 @@ void Usage(const BuildConfig& config) { " -d MODE enable debugging (use -d list to list modes)\n" " -t TOOL run a subtool (use -t list to list subtools)\n" " terminates toplevel options; further flags are passed to the tool\n" -" -w FLAG adjust warnings (use -w list to list warnings)\n", - kNinjaVersion, config.parallelism); +" -w FLAG adjust warnings (use -w list to list warnings)\n" +#ifndef _WIN32 +"\n" +" --frontend COMMAND execute COMMAND and pass serialized build output to it\n" +#endif + , kNinjaVersion, config.parallelism); } /// Choose a default value for the -j (parallelism) flag. @@ -1034,8 +1038,14 @@ int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { config->parallelism = GuessParallelism(); - enum { OPT_VERSION = 1 }; + enum { + OPT_VERSION = 1, + OPT_FRONTEND = 2, + }; const option kLongOptions[] = { +#ifndef _WIN32 + { "frontend", required_argument, NULL, OPT_FRONTEND }, +#endif { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, { NULL, 0, NULL, 0 } @@ -1043,7 +1053,7 @@ int ReadFlags(int* argc, char*** argv, int opt; while (!options->tool && - (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions, + (opt = getopt_long(*argc, *argv, "d:f:j:k:l:mnt:vw:C:h", kLongOptions, NULL)) != -1) { switch (opt) { case 'd': @@ -1102,6 +1112,9 @@ int ReadFlags(int* argc, char*** argv, case OPT_VERSION: printf("%s\n", kNinjaVersion); return 0; + case OPT_FRONTEND: + config->frontend = optarg; + break; case 'h': default: Usage(*config); @@ -1147,13 +1160,22 @@ int real_main(int argc, char** argv) { return (ninja.*options.tool->func)(&options, argc, argv); } - Status* status = new StatusPrinter(config); + Status* status = NULL; // Limit number of rebuilds, to prevent infinite loops. const int kCycleLimit = 100; for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { NinjaMain ninja(ninja_command, config); + if (status == NULL) { +#ifndef _WIN32 + if (config.frontend != NULL) + status = new StatusSerializer(config); + else +#endif + status = new StatusPrinter(config); + } + ManifestParser parser(&ninja.state_, &ninja.disk_interface_, options.dupe_edges_should_err ? kDupeEdgeActionError @@ -1192,11 +1214,14 @@ int real_main(int argc, char** argv) { int result = ninja.RunBuild(argc, argv, status); if (g_metrics) ninja.DumpMetrics(); + + delete status; return result; } status->Error("manifest '%s' still dirty after %d tries", options.input_file, kCycleLimit); + delete status; return 1; } diff --git a/src/status.cc b/src/status.cc index 1d495f2e4b..7860b9a1bb 100644 --- a/src/status.cc +++ b/src/status.cc @@ -14,8 +14,9 @@ #include "status.h" -#include +#include #include +#include StatusPrinter::StatusPrinter(const BuildConfig& config) : config_(config), @@ -49,7 +50,7 @@ void StatusPrinter::BuildEdgeStarted(Edge* edge, int64_t start_time_millis) { } void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, - bool success, const string& output) { + const CommandRunner::Result* result) { time_millis_ = end_time_millis; ++finished_edges_; @@ -65,7 +66,7 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, --running_edges_; // Print the command that is spewing before printing its output. - if (!success) { + if (!result->success()) { string outputs; for (vector::const_iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) @@ -75,7 +76,7 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); } - if (!output.empty()) { + if (!result->output.empty()) { // ninja sets stdout and stderr of subprocesses to a pipe, to be able to // check if the output is empty. Some compilers, e.g. clang, check // isatty(stderr) to decide if they should print colored output. @@ -90,9 +91,9 @@ void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, // TODO: There should be a flag to disable escape code stripping. string final_output; if (!printer_.is_smart_terminal()) - final_output = StripAnsiEscapeCodes(output); + final_output = StripAnsiEscapeCodes(result->output); else - final_output = output; + final_output = result->output; #ifdef _WIN32 // Fix extra CR being added on Windows, writing out CR CR LF (#773) @@ -237,3 +238,152 @@ void StatusPrinter::Info(const char* msg, ...) { ::Info(msg, ap); va_end(ap); } + +#ifndef _WIN32 + +#include "frontend.pb.h" +#include "proto.h" + +StatusSerializer::StatusSerializer(const BuildConfig& config) : + config_(config), subprocess_(NULL), total_edges_(0) { + int output_pipe[2]; + if (pipe(output_pipe) < 0) + Fatal("pipe: %s", strerror(errno)); + SetCloseOnExec(output_pipe[1]); + + f_ = fdopen(output_pipe[1], "wb"); + setvbuf(f_, NULL, _IONBF, 0); + filebuf_ = new ofilebuf(f_); + out_ = new std::ostream(filebuf_); + + // Launch the frontend as a subprocess with write-end of the pipe as fd 3 + subprocess_ = subprocess_set_.Add(config.frontend, /*use_console=*/true, + output_pipe[0]); + close(output_pipe[0]); +} + +StatusSerializer::~StatusSerializer() { + delete out_; + delete filebuf_; + fclose(f_); + subprocess_->Finish(); + subprocess_set_.Clear(); +} + +void StatusSerializer::Send() { + // Send the proto as a length-delimited message + WriteVarint32NoTag(out_, proto_.ByteSizeLong()); + proto_.SerializeToOstream(out_); + proto_.Clear(); + out_->flush(); +} + +void StatusSerializer::PlanHasTotalEdges(int total) { + if (total_edges_ != total) { + total_edges_ = total; + ninja::Status::TotalEdges *total_edges = proto_.mutable_total_edges(); + total_edges->set_total_edges(total_edges_); + Send(); + } +} + +void StatusSerializer::BuildEdgeStarted(Edge* edge, int64_t start_time_millis) { + ninja::Status::EdgeStarted* edge_started = proto_.mutable_edge_started(); + + edge_started->set_id(edge->id_); + edge_started->set_start_time(start_time_millis); + + edge_started->mutable_inputs()->reserve(edge->inputs_.size()); + for (vector::iterator it = edge->inputs_.begin(); + it != edge->inputs_.end(); ++it) { + edge_started->add_inputs((*it)->path()); + } + + edge_started->mutable_outputs()->reserve(edge->inputs_.size()); + for (vector::iterator it = edge->outputs_.begin(); + it != edge->outputs_.end(); ++it) { + edge_started->add_outputs((*it)->path()); + } + + edge_started->set_desc(edge->GetBinding("description")); + + edge_started->set_command(edge->GetBinding("command")); + + edge_started->set_console(edge->use_console()); + + Send(); +} + +void StatusSerializer::BuildEdgeFinished(Edge* edge, int64_t end_time_millis, + const CommandRunner::Result* result) { + ninja::Status::EdgeFinished* edge_finished = proto_.mutable_edge_finished(); + + edge_finished->set_id(edge->id_); + edge_finished->set_end_time(end_time_millis); + edge_finished->set_status(result->status); + edge_finished->set_output(result->output); + + Send(); +} + +void StatusSerializer::BuildStarted() { + ninja::Status::BuildStarted* build_started = proto_.mutable_build_started(); + + build_started->set_parallelism(config_.parallelism); + build_started->set_verbose((config_.verbosity == BuildConfig::VERBOSE)); + + Send(); +} + +void StatusSerializer::BuildFinished() { + proto_.mutable_build_finished(); + Send(); +} + +void StatusSerializer::Message(ninja::Status::Message::Level level, + const char* msg, va_list ap) { + va_list ap2; + va_copy(ap2, ap); + + int len = vsnprintf(NULL, 0, msg, ap2); + if (len < 0) { + Fatal("vsnprintf failed"); + } + + va_end(ap2); + + string buf; + buf.resize(len + 1); + + len = vsnprintf(&buf[0], len + 1, msg, ap); + buf.resize(len); + + ninja::Status::Message* message = proto_.mutable_message(); + + message->set_level(level); + message->set_message(buf); + + Send(); +} + +void StatusSerializer::Info(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + Message(ninja::Status::Message::INFO, msg, ap); + va_end(ap); +} + +void StatusSerializer::Warning(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + Message(ninja::Status::Message::WARNING, msg, ap); + va_end(ap); +} + +void StatusSerializer::Error(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + Message(ninja::Status::Message::ERROR, msg, ap); + va_end(ap); +} +#endif // !_WIN32 diff --git a/src/status.h b/src/status.h index a7d89648ed..73073c6194 100644 --- a/src/status.h +++ b/src/status.h @@ -15,12 +15,15 @@ #ifndef NINJA_STATUS_H_ #define NINJA_STATUS_H_ +#include + #include #include using namespace std; #include "build.h" #include "line_printer.h" +#include "subprocess.h" /// Abstract interface to object that tracks the status of a build: /// completion fraction, printing updates. @@ -28,7 +31,7 @@ struct Status { virtual void PlanHasTotalEdges(int total) = 0; virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis) = 0; virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, - bool success, const string& output) = 0; + const CommandRunner::Result* result) = 0; virtual void BuildStarted() = 0; virtual void BuildFinished() = 0; @@ -46,7 +49,7 @@ struct StatusPrinter : Status { virtual void PlanHasTotalEdges(int total); virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis); virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, - bool success, const string& output); + const CommandRunner::Result* result); virtual void BuildStarted(); virtual void BuildFinished(); @@ -113,4 +116,45 @@ struct StatusPrinter : Status { mutable SlidingRateInfo current_rate_; }; +#ifndef _WIN32 + +#include "filebuf.h" +#include "frontend.pb.h" + +/// Implementation of the Status interface that serializes the status as +/// protocol buffer messages to a subprocess +struct StatusSerializer : Status { + explicit StatusSerializer(const BuildConfig& config); + virtual ~StatusSerializer(); + + virtual void PlanHasTotalEdges(int total); + virtual void BuildEdgeStarted(Edge* edge, int64_t start_time); + virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis, + const CommandRunner::Result* result); + virtual void BuildStarted(); + virtual void BuildFinished(); + + virtual void Info(const char* msg, ...); + virtual void Warning(const char* msg, ...); + virtual void Error(const char* msg, ...); + + const BuildConfig& config_; + + FILE* f_; + ofilebuf *filebuf_; + std::ostream *out_; + + ninja::Status proto_; + + SubprocessSet subprocess_set_; + Subprocess* subprocess_; + + int total_edges_; +private: + void Message(ninja::Status::Message::Level level, const char* msg, va_list ap); + void Send(); +}; + +#endif // !_WIN32 + #endif // NINJA_STATUS_H_ diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index 1de22c38f7..ea1ab98b1c 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -40,7 +40,8 @@ Subprocess::~Subprocess() { Finish(); } -bool Subprocess::Start(SubprocessSet* set, const string& command) { +bool Subprocess::Start(SubprocessSet* set, const string& command, + int extra_fd) { int output_pipe[2]; if (pipe(output_pipe) < 0) Fatal("pipe: %s", strerror(errno)); @@ -60,6 +61,11 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { if (posix_spawn_file_actions_addclose(&action, output_pipe[0]) != 0) Fatal("posix_spawn_file_actions_addclose: %s", strerror(errno)); + if (extra_fd >= 0) { + if (posix_spawn_file_actions_adddup2(&action, extra_fd, 3) != 0) + Fatal("posix_spawn_file_actions_adddup2: %s", strerror(errno)); + } + posix_spawnattr_t attr; if (posix_spawnattr_init(&attr) != 0) Fatal("posix_spawnattr_init: %s", strerror(errno)); @@ -208,9 +214,10 @@ SubprocessSet::~SubprocessSet() { Fatal("sigprocmask: %s", strerror(errno)); } -Subprocess *SubprocessSet::Add(const string& command, bool use_console) { +Subprocess *SubprocessSet::Add(const string& command, bool use_console, + int extra_fd) { Subprocess *subprocess = new Subprocess(use_console); - if (!subprocess->Start(this, command)) { + if (!subprocess->Start(this, command, extra_fd)) { delete subprocess; return 0; } diff --git a/src/subprocess.h b/src/subprocess.h index b2d486ca40..da7c80dd21 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -53,7 +53,7 @@ struct Subprocess { private: Subprocess(bool use_console); - bool Start(struct SubprocessSet* set, const string& command); + bool Start(struct SubprocessSet* set, const string& command, int extra_fd); void OnPipeReady(); string buf_; @@ -84,7 +84,8 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess* Add(const string& command, bool use_console = false); + Subprocess* Add(const string& command, bool use_console = false, + int extra_fd = -1); bool DoWork(); Subprocess* NextFinished(); void Clear();