Add frontend option for build status output

Add a --frontend option that takes a command that will handle printing
build status.  The command will be executed with the read of a pipe on
fd 3, and each build event will cause a serialized message to be sent
on the pipe.

The frontend/ directory includes a description of the interface
between Ninja and a frontend, and an implementation of the interface
in python that mimics the native interface in ninja, which can be run
with:
ninja --frontend=frontend/native.py

configure.py

@@ -272,6 +272,8 @@ def _run_command(self, cmdline):
 
 def src(filename):
     return os.path.join('$root', 'src', filename)
+def frontend(filename):
+    return os.path.join('$root', 'frontend', filename)
 def built(filename):
     return os.path.join('$builddir', filename)
 def doc(filename):
@@ -475,6 +477,43 @@ def has_re2c():
            "changes to src/*.in.cc will not affect your build.")
 n.newline()
 
+n.comment('the proto descriptor is generated using protoc.')
+def has_protoc():
+    try:
+        proc = subprocess.Popen(['protoc', '--version'], stdout=subprocess.PIPE)
+        return proc.communicate()[0].startswith("libprotoc")
+    except OSError:
+        return False
+
+def can_generate_proto_header():
+    try:
+        tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+        proc = subprocess.Popen([tool, '--probe'], stdout=subprocess.PIPE)
+        return proc.communicate()[0].startswith("ok")
+    except OSError:
+        return False
+
+if has_protoc() and can_generate_proto_header():
+    # Use protoc to write out frontend.proto converted to a descriptor proto
+    n.rule('protoc',
+           command='protoc $in -o $out',
+           description='PROTOC $out')
+    n.build(frontend('frontend.pb'), 'protoc', src('frontend.proto'))
+
+    # Use generate_proto_header.py to read in the descriptor proto and write
+    # a header containing field numbers and types.
+    n.rule('generate_proto_header',
+           command='$tool $in $out',
+           description='GEN $out')
+    # Generate the .h file in the source directory so we can check them in.
+    tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+    n.build(src('frontend.pb.h'), 'generate_proto_header', frontend('frontend.pb'),
+            implicit=[tool], variables=[('tool', tool)])
+else:
+    print("warning: A version of protoc or the python protobuf library was not found; "
+           "changes to src/frontend.proto will not affect your build.")
+n.newline()
+
 n.comment('Core source files all build into ninja library.')
 for name in ['build',
              'build_log',
@@ -498,7 +537,7 @@ def has_re2c():
              'string_piece_util',
              'util',
              'version']:
-    objs += cxx(name)
+    objs += cxx(name, order_only=src('frontend.pb.h'))
 if platform.is_windows():
     for name in ['subprocess-win32',
                  'includes_normalize-win32',

frontend/FRONTEND.md

@@ -0,0 +1,54 @@
+Ninja Frontend Interface
+========================
+
+Ninja can use another program as a frontend to display build status information.
+This document describes the interface between Ninja and the frontend.
+
+Connecting
+----------
+
+The frontend is passed to Ninja using a --frontend argument.  The argument is
+executed the same as a build rule Ninja, wrapped in `sh -c` on Linux.  The
+frontend will be executed with the read end of a pipe open on file descriptor
+`3`.
+
+Ninja will pass [Protocol Buffers](https://developers.google.com/protocol-buffers/) generated from src/frontend.proto.
+
+stdin/stdout/stderr
+-------------------
+
+The frontend will have stdin, stdout, and stderr connected to the same file
+descriptors as Ninja.  The frontend MAY read from stdin, however, if it does,
+it MUST NOT read from stdin whenever a job in the console pool is running,
+from when an `EdgeStarted` message is received with the `use_console` value
+set to `true`, to when an `EdgeFinished` message is received with the same value
+for `id`.  Console rules may write directly to the same stdout/stderr as the
+frontend.
+
+Exiting
+-------
+
+The frontend MUST exit when the input pipe on fd `3` is closed.  When a build
+finishes, either successfully, due to error, or on interrupt, Ninja will close
+the pipe and then block until the frontend exits.
+
+Experimenting with frontends
+----------------------------
+
+To run Ninja with a frontend that mimics the behavior of Ninja's normal output:
+```
+$ ./ninja --frontend=frontend/native.py
+```
+
+To save serialized output to a file:
+```
+$ ./ninja --frontend='cat /proc/self/fd/3 > ninja.pb all
+```
+
+To run a frontend with serialized input from a file:
+```
+$ frontend/native.py 3< ninja.pb
+```
+
+The serialized output of a clean Ninja build is included in `frontend/ninja.pb`.
+

frontend/dump.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import sys
+
+import frontend
+
+def main():
+    if len(sys.argv) >= 2:
+        f = open(sys.argv[1], 'rb')
+    else:
+        f = frontend.default_reader()
+
+    for msg in frontend.Frontend(f):
+        print('---------------------------------')
+        sys.stdout.write(str(msg))
+
+if __name__ == '__main__':
+    main()
+

frontend/frontend.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+"""Ninja frontend interface.
+
+This module implements a Ninja frontend interface that delegates handling each
+message to a handler object
+"""
+
+import os
+import sys
+
+import google.protobuf.descriptor_pb2
+import google.protobuf.message_factory
+
+def default_reader():
+    fd = 3
+    return os.fdopen(fd, 'rb', 0)
+
+class Frontend(object):
+    """Generator class that parses length-delimited ninja status messages
+    through a ninja frontend interface.
+    """
+
+    def __init__(self, reader=default_reader()):
+        self.reader = reader
+        self.status_class = self.get_status_proto()
+
+    def get_status_proto(self):
+        fd_set = google.protobuf.descriptor_pb2.FileDescriptorSet()
+        descriptor = os.path.join(os.path.dirname(__file__), 'frontend.pb')
+        with open(descriptor, 'rb') as f:
+            fd_set.ParseFromString(f.read())
+
+        if len(fd_set.file) != 1:
+            raise RuntimeError('expected exactly one file descriptor in ' + descriptor)
+
+        messages = google.protobuf.message_factory.GetMessages(fd_set.file)
+        return messages['ninja.Status']
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        size = 0
+        shift = 0
+        while True:
+            byte = bytearray(self.reader.read(1))
+            if len(byte) == 0:
+                raise StopIteration()
+
+            byte = byte[0]
+            size += (byte & 0x7f) << (shift * 7)
+            if (byte & 0x80) == 0:
+                break
+            shift += 1
+            if shift > 4:
+                raise RuntimeError('Expected varint32 length-delimeted message')
+
+        message = self.reader.read(size)
+        if len(message) != size:
+            raise EOFError('Unexpected EOF reading %d bytes' % size)
+
+        return self.status_class.FromString(message)