Merge upstream commit '0a2e2cae' into master
* commit '0a2e2cae7038ce519b0524c07d7135c3e520c9cd':
Restore depfile toleration of multiple output paths on distinct lines
Fix depfile parser handling of multiple rules
Fix depfile parser test case line continuation
Re-arrange depfile parser token processing logic
Re-generate depfile parser with re2cc 1.0.1
depfile_parser.cc conflicts were handled by regenerating with re2c
The other conflicts were all fairly simple argument/option ordering.
Test: compare .ninja_deps of a full android build before and after this
change, they're equivalent
Change-Id: I74b7fc56e035c9dbab4c987b6af17329c596a898
diff --git a/Android.bp b/Android.bp
new file mode 100644
index 0000000..1c3ed4f
--- /dev/null
+++ b/Android.bp
@@ -0,0 +1,133 @@
+// Copyright 2016 Google Inc. All rights reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+cc_defaults {
+ name: "ninja_defaults",
+ cflags: [
+ "-Wall",
+ "-Wextra",
+ "-Wno-deprecated",
+ "-Wno-missing-field-initializers",
+ "-Wno-unused-parameter",
+ "-fno-exceptions",
+ "-fvisibility=hidden",
+ "-DNINJA_PYTHON=\"python\"",
+ "-DNINJA_HAVE_BROWSE",
+ "-UNDEBUG",
+ ],
+ target: {
+ linux_glibc: {
+ cflags: ["-DUSE_PPOLL"],
+ }
+ }
+}
+
+genrule {
+ name: "ninja_browse_py",
+
+ cmd: "$(location src/inline.sh) kBrowsePy <$(in) >$(out)",
+ tool_files: ["src/inline.sh"],
+
+ srcs: ["src/browse.py"],
+ out: ["build/browse_py.h"],
+}
+
+cc_library_host_static {
+ name: "libninja",
+ defaults: ["ninja_defaults"],
+ generated_headers: ["ninja_browse_py"],
+ srcs: [
+ "src/build.cc",
+ "src/build_log.cc",
+ "src/clean.cc",
+ "src/clparser.cc",
+ "src/debug_flags.cc",
+ "src/depfile_parser.cc",
+ "src/deps_log.cc",
+ "src/disk_interface.cc",
+ "src/edit_distance.cc",
+ "src/eval_env.cc",
+ "src/graph.cc",
+ "src/graphviz.cc",
+ "src/lexer.cc",
+ "src/line_printer.cc",
+ "src/manifest_chunk_parser.cc",
+ "src/manifest_parser.cc",
+ "src/metrics.cc",
+ "src/proto.cc",
+ "src/state.cc",
+ "src/status.cc",
+ "src/thread_pool.cc",
+ "src/util.cc",
+ "src/version.cc",
+ "src/browse.cc",
+ "src/subprocess-posix.cc",
+ ],
+}
+
+cc_binary_host {
+ name: "ninja",
+ defaults: ["ninja_defaults"],
+ srcs: ["src/ninja.cc"],
+ static_libs: ["libninja"],
+
+ // Use jemalloc for better multithreaded allocation performance. e.g. Using
+ // jemalloc can improve the overall runtime by 10x vs the default allocator.
+ target: {
+ linux_glibc: {
+ shared_libs: ["libjemalloc"],
+ },
+ },
+}
+
+cc_test_host {
+ name: "ninja_test",
+ defaults: ["ninja_defaults"],
+ static_libs: ["libninja"],
+ gtest: false,
+ srcs: [
+ "src/build_log_test.cc",
+ "src/build_test.cc",
+ "src/clean_test.cc",
+ "src/clparser_test.cc",
+ "src/depfile_parser_test.cc",
+ "src/deps_log_test.cc",
+ "src/disk_interface_test.cc",
+ "src/edit_distance_test.cc",
+ "src/graph_test.cc",
+ "src/lexer_test.cc",
+ "src/manifest_parser_test.cc",
+ "src/ninja_test.cc",
+ "src/state_test.cc",
+ "src/status_test.cc",
+ "src/subprocess_test.cc",
+ "src/test.cc",
+ "src/util_test.cc",
+ ],
+}
+
+cc_test_host {
+ name: "ninja_tests",
+ gtest: false,
+ defaults: ["ninja_defaults"],
+ test_per_src: true,
+ static_libs: ["libninja"],
+ srcs: [
+ "src/build_log_perftest.cc",
+ "src/canon_perftest.cc",
+ "src/depfile_parser_perftest.cc",
+ "src/hash_collision_bench.cc",
+ "src/manifest_parser_perftest.cc",
+ ],
+}
diff --git a/configure.py b/configure.py
index 78cd1de..0b05d06 100755
--- a/configure.py
+++ b/configure.py
@@ -272,6 +272,8 @@
def src(filename):
return os.path.join('$root', 'src', filename)
+def frontend(filename):
+ return os.path.join('$root', 'frontend', filename)
def built(filename):
return os.path.join('$builddir', filename)
def doc(filename):
@@ -328,7 +330,8 @@
cflags += ['/Ox', '/DNDEBUG', '/GL']
ldflags += ['/LTCG', '/OPT:REF', '/OPT:ICF']
else:
- cflags = ['-g', '-Wall', '-Wextra',
+ cflags = ['-std=c++11',
+ '-g', '-Wall', '-Wextra',
'-Wno-deprecated',
'-Wno-missing-field-initializers',
'-Wno-unused-parameter',
@@ -364,7 +367,10 @@
libs = []
-if platform.is_mingw():
+if platform.is_linux():
+ cflags.append('-pthread')
+ ldflags.append('-pthread')
+elif platform.is_mingw():
cflags.remove('-fvisibility=hidden');
ldflags.append('-static')
elif platform.is_solaris():
@@ -484,6 +490,43 @@
"changes to src/*.in.cc will not affect your build.")
n.newline()
+n.comment('the proto descriptor is generated using protoc.')
+def has_protoc():
+ try:
+ proc = subprocess.Popen(['protoc', '--version'], stdout=subprocess.PIPE)
+ return proc.communicate()[0].startswith("libprotoc")
+ except OSError:
+ return False
+
+def can_generate_proto_header():
+ try:
+ tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+ proc = subprocess.Popen([tool, '--probe'], stdout=subprocess.PIPE)
+ return proc.communicate()[0].startswith("ok")
+ except OSError:
+ return False
+
+if has_protoc() and can_generate_proto_header():
+ # Use protoc to write out frontend.proto converted to a descriptor proto
+ n.rule('protoc',
+ command='protoc $in -o $out',
+ description='PROTOC $out')
+ n.build(frontend('frontend.pb'), 'protoc', src('frontend.proto'))
+
+ # Use generate_proto_header.py to read in the descriptor proto and write
+ # a header containing field numbers and types.
+ n.rule('generate_proto_header',
+ command='$tool $in $out',
+ description='GEN $out')
+ # Generate the .h file in the source directory so we can check them in.
+ tool = os.path.join(sourcedir, 'misc', 'generate_proto_header.py')
+ n.build(src('frontend.pb.h'), 'generate_proto_header', frontend('frontend.pb'),
+ implicit=[tool], variables=[('tool', tool)])
+else:
+ print("warning: A version of protoc or the python protobuf library was not found; "
+ "changes to src/frontend.proto will not affect your build.")
+n.newline()
+
n.comment('Core source files all build into ninja library.')
cxxvariables = []
if platform.is_msvc():
@@ -502,13 +545,17 @@
'graphviz',
'lexer',
'line_printer',
+ 'manifest_chunk_parser',
'manifest_parser',
'metrics',
+ 'proto',
'state',
+ 'status',
'string_piece_util',
+ 'thread_pool',
'util',
'version']:
- objs += cxx(name, variables=cxxvariables)
+ objs += cxx(name, order_only=src('frontend.pb.h'), variables=cxxvariables)
if platform.is_windows():
for name in ['subprocess-win32',
'includes_normalize-win32',
@@ -570,6 +617,7 @@
'manifest_parser_test',
'ninja_test',
'state_test',
+ 'status_test',
'string_piece_util_test',
'subprocess_test',
'test',
diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index 7b1c3ba..4f8c74f 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -1,5 +1,6 @@
The Ninja build system
======================
+v1.7.1, Apr 2016
Introduction
@@ -695,6 +696,8 @@
Order-only dependencies may be tacked on the end with +||
_dependency1_ _dependency2_+. (See <<ref_dependencies,the reference on
dependency types>>.)
+ Validations may be taked on the end with +|@ _validation1_ _validation2_+.
+ (See <<validations,the reference on validations>>.)
+
Implicit outputs _(available since Ninja 1.7)_ may be added before
the `:` with +| _output1_ _output2_+ and do not appear in `$out`.
@@ -815,6 +818,31 @@
the full command or its description; if a command fails, the full command
line will always be printed before the command's output.
+`phony_output`:: _(Android-specific patch)_ if present, Ninja will not attempt
+ to look for them on disk. These rules are considered always dirty, and will
+ run every time they're depended upon. This behavior is very similar to Make's
+ `.PHONY` concept.
++
+This can be similar to the `phony` rule, but can have an attached `command`.
+`phony` rules are also only considered dirty in two cases: if their inputs are
+dirty, or if they have no inputs and a file with the same name does not exist on
+disk.
++
+Other build rules may not depend on `phony_output` rules unless they are also
+`phony_output`, so that it's not possible to accidentally cause everything to
+rebuild on every run.
++
+When `-o usesphonyoutputs=yes` is set on the ninja command line, it becomes an
+error for a `phony` rule to cause rebuilds, so that users can be found and
+migrated.
++
+Properly using `phony_output` and turning on `-o usesphonyoutputs=yes` allows
+the `-w outputdir={err,warn}` (consider output files that are directories as
+errors/warnings), `-w missingoutfile={err,warn}` (error/warn when an output file
+does not exist after a successful rule execution), and `-w oldoutput={err,warn}`
+(error/warn when an output file is not updated after a successful non-restat
+rule execution) flags to function.
+
`generator`:: if present, specifies that this rule is used to
re-invoke the generator program. Files built using `generator`
rules are treated specially in two ways: firstly, they will not be
@@ -946,6 +974,31 @@
File paths are compared as is, which means that an absolute path and a
relative path, pointing to the same file, are considered different by Ninja.
+[[validations]]
+Validations
+~~~~~~~~~~~
+Validations listed on the build line cause the specified files to be
+added to the top level of the build graph (as if they were specified
+on the Ninja command line) whenever the build line is a transitive
+dependency of one of the targets specified on the command line or a
+default target.
+
+Validations are added to the build graph regardless of whether the output
+files of the build statement are dirty are not, and the dirty state of
+the build statement that outputs the file being used as a validation
+has no effect on the dirty state of the build statement that requested it.
+
+A build edge can list another build edge as a validation even if the second
+edge depends on the first.
+
+Validations are designed to handle rules that perform error checking but
+don't produce any artifacts needed by the build, for example static
+analysis tools. Marking the static analysis rule as an implicit input
+of the main build rule of the source files or of the rules that depend
+on the main build rule would slow down the critical path of the build,
+but using a validation would allow the build to proceed in parallel with
+the static analysis rule once the main build rule is complete.
+
Variable expansion
~~~~~~~~~~~~~~~~~~
diff --git a/frontend/FRONTEND.md b/frontend/FRONTEND.md
new file mode 100644
index 0000000..5ff6e32
--- /dev/null
+++ b/frontend/FRONTEND.md
@@ -0,0 +1,51 @@
+Ninja Frontend Interface
+========================
+
+Ninja can use another program as a frontend to display build status information.
+This document describes the interface between Ninja and the frontend.
+
+Connecting
+----------
+
+The frontend is passed to Ninja using a --frontend argument. The argument is
+executed the same as a build rule Ninja, wrapped in `sh -c` on Linux. The
+frontend will be executed with the read end of a pipe open on file descriptor
+`3`.
+
+Ninja will pass [Protocol Buffers](https://developers.google.com/protocol-buffers/) generated from src/frontend.proto.
+
+stdin/stdout/stderr
+-------------------
+
+The frontend will have stdin, stdout, and stderr connected to the same file
+descriptors as Ninja. The frontend MAY read from stdin, however, if it does,
+it MUST NOT read from stdin whenever a job in the console pool is running,
+from when an `EdgeStarted` message is received with the `use_console` value
+set to `true`, to when an `EdgeFinished` message is received with the same value
+for `id`. Console rules may write directly to the same stdout/stderr as the
+frontend.
+
+Exiting
+-------
+
+The frontend MUST exit when the input pipe on fd `3` is closed. When a build
+finishes, either successfully, due to error, or on interrupt, Ninja will close
+the pipe and then block until the frontend exits.
+
+Experimenting with frontends
+----------------------------
+
+To run Ninja with a frontend that mimics the behavior of Ninja's normal output:
+```
+$ ./ninja --frontend=frontend/native.py
+```
+
+To save serialized output to a file:
+```
+$ ./ninja --frontend='cat <&3 > ninja.pb' all
+```
+
+To run a frontend with serialized input from a file:
+```
+$ frontend/native.py 3< ninja.pb
+```
diff --git a/frontend/dump.py b/frontend/dump.py
new file mode 100755
index 0000000..d276fd0
--- /dev/null
+++ b/frontend/dump.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import sys
+
+import frontend
+
+def main():
+ if len(sys.argv) >= 2:
+ f = open(sys.argv[1], 'rb')
+ else:
+ f = frontend.default_reader()
+
+ for msg in frontend.Frontend(f):
+ print('---------------------------------')
+ sys.stdout.write(str(msg))
+
+if __name__ == '__main__':
+ main()
+
diff --git a/frontend/frontend.pb b/frontend/frontend.pb
new file mode 100644
index 0000000..718f173
--- /dev/null
+++ b/frontend/frontend.pb
Binary files differ
diff --git a/frontend/frontend.py b/frontend/frontend.py
new file mode 100755
index 0000000..ac8fa25
--- /dev/null
+++ b/frontend/frontend.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+"""Ninja frontend interface.
+
+This module implements a Ninja frontend interface that delegates handling each
+message to a handler object
+"""
+
+import os
+import sys
+
+import google.protobuf.descriptor_pb2
+import google.protobuf.message_factory
+
+def default_reader():
+ fd = 3
+ return os.fdopen(fd, 'rb', 0)
+
+class Frontend(object):
+ """Generator class that parses length-delimited ninja status messages
+ through a ninja frontend interface.
+ """
+
+ def __init__(self, reader=default_reader()):
+ self.reader = reader
+ self.status_class = self.get_status_proto()
+
+ def get_status_proto(self):
+ fd_set = google.protobuf.descriptor_pb2.FileDescriptorSet()
+ descriptor = os.path.join(os.path.dirname(__file__), 'frontend.pb')
+ with open(descriptor, 'rb') as f:
+ fd_set.ParseFromString(f.read())
+
+ if len(fd_set.file) != 1:
+ raise RuntimeError('expected exactly one file descriptor in ' + descriptor)
+
+ messages = google.protobuf.message_factory.GetMessages(fd_set.file)
+ return messages['ninja.Status']
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return self.next()
+
+ def next(self):
+ size = 0
+ shift = 0
+ while True:
+ byte = bytearray(self.reader.read(1))
+ if len(byte) == 0:
+ raise StopIteration()
+
+ byte = byte[0]
+ size += (byte & 0x7f) << (shift * 7)
+ if (byte & 0x80) == 0:
+ break
+ shift += 1
+ if shift > 4:
+ raise RuntimeError('Expected varint32 length-delimeted message')
+
+ message = self.reader.read(size)
+ if len(message) != size:
+ raise EOFError('Unexpected EOF reading %d bytes' % size)
+
+ return self.status_class.FromString(message)
diff --git a/frontend/native.py b/frontend/native.py
new file mode 100755
index 0000000..4862268
--- /dev/null
+++ b/frontend/native.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import collections
+import ctypes
+import os
+import re
+import struct
+import sys
+
+import frontend
+
+class SlidingRateInfo(object):
+ def __init__(self, n=32):
+ self.rate = -1
+ self.last_update = -1
+ self.times = collections.deque(maxlen=n)
+
+ def update_rate(self, update_hint, time_millis):
+ if update_hint == self.last_update:
+ return
+
+ self.last_update = update_hint
+
+ if len(self.times) == self.times.maxlen:
+ self.times.popleft()
+ self.times.append(time_millis)
+ if self.times[-1] != self.times[0]:
+ self.rate = len(self.times) / ((self.times[-1] - self.times[0]) / 1e3)
+
+strip_ansi_re = re.compile(r'\x1B\[[^a-zA-Z]*[a-zA-Z]')
+def strip_ansi_escape_codes(output):
+ return strip_ansi_re.sub('', output)
+
+class NinjaNativeFrontend:
+ def __init__(self):
+ self.total_edges = 0
+ self.running_edges = 0
+ self.started_edges = 0
+ self.finished_edges = 0
+ self.running = {}
+
+ self.time_millis = 0
+
+ self.progress_status_format = os.getenv('NINJA_STATUS', '[%f/%t] ')
+ self.current_rate = SlidingRateInfo()
+ self.console_locked = False
+
+ self.printer = LinePrinter()
+ self.verbose = False
+
+ def handle(self, msg):
+ handled = False
+ if msg.HasField('total_edges'):
+ handled = True
+ self.total_edges = msg.total_edges.total_edges
+
+ if msg.HasField('build_started'):
+ handled = True
+ self.verbose = msg.build_started.verbose
+ self.current_rate = SlidingRateInfo(msg.build_started.parallelism)
+ self.running_edges = 0
+ self.started_edges = 0
+ self.finished_edges = 0
+ self.running = {}
+
+ if msg.HasField('build_finished'):
+ handled = True
+ self.printer.set_console_locked(False)
+ self.printer.print_on_new_line('')
+
+ if msg.HasField('edge_started'):
+ handled = True
+ self.started_edges += 1
+ self.running_edges += 1
+ self.running[msg.edge_started.id] = msg.edge_started
+ self.time_millis = msg.edge_started.start_time
+ if msg.edge_started.console or self.printer.smart_terminal:
+ self.print_status(msg.edge_started)
+ if msg.edge_started.console:
+ self.printer.set_console_locked(True)
+
+ if msg.HasField('edge_finished'):
+ handled = True
+ self.finished_edges += 1
+ self.time_millis = msg.edge_finished.end_time
+
+ edge_started = self.running[msg.edge_finished.id]
+
+ if edge_started.console:
+ self.printer.set_console_locked(False)
+
+ if not edge_started.console:
+ self.print_status(edge_started)
+
+ self.running_edges -= 1
+ del self.running[msg.edge_finished.id]
+
+ # Print the command that is spewing before printing its output.
+ if msg.edge_finished.status != 0:
+ self.printer.print_on_new_line('FAILED: ' + ' '.join(edge_started.outputs))
+ self.printer.print_on_new_line(edge_started.command)
+
+ # ninja sets stdout and stderr of subprocesses to a pipe, to be able to
+ # check if the output is empty. Some compilers, e.g. clang, check
+ # isatty(stderr) to decide if they should print colored output.
+ # To make it possible to use colored output with ninja, subprocesses should
+ # be run with a flag that forces them to always print color escape codes.
+ # To make sure these escape codes don't show up in a file if ninja's output
+ # is piped to a file, ninja strips ansi escape codes again if it's not
+ # writing to a |smart_terminal_|.
+ # (Launching subprocesses in pseudo ttys doesn't work because there are
+ # only a few hundred available on some systems, and ninja can launch
+ # thousands of parallel compile commands.)
+ # TODO: There should be a flag to disable escape code stripping.
+ if msg.edge_finished.output != '':
+ if not self.printer.smart_terminal:
+ msg.edge_finished.output = strip_ansi_escape_codes(msg.edge_finished.output)
+ self.printer.print_on_new_line(msg.edge_finished.output)
+
+ if msg.HasField('message'):
+ handled = True
+ # TODO(colincross): get the enum values from proto
+ if msg.message.level == 0:
+ prefix = 'ninja: '
+ elif msg.message.level == 1:
+ prefix = 'ninja: warning: '
+ elif msg.message.level == 2:
+ prefix = 'ninja: error: '
+ else
+ prefix = ''
+ self.printer.print_line(prefix + msg.message.message, LinePrinter.LINE_FULL)
+
+ if not handled:
+ pass
+
+
+ def format_progress_status(self, fmt):
+ out = ''
+ fmt_iter = iter(fmt)
+ for c in fmt_iter:
+ if c == '%':
+ c = next(fmt_iter)
+ if c == '%':
+ out += c
+ elif c == 's':
+ out += str(self.started_edges)
+ elif c == 't':
+ out += str(self.total_edges)
+ elif c == 'r':
+ out += str(self.running_edges)
+ elif c == 'u':
+ out += str(self.total_edges - self.started_edges)
+ elif c == 'f':
+ out += str(self.finished_edges)
+ elif c == 'o':
+ if self.time_millis > 0:
+ rate = self.finished_edges / (self.time_millis / 1e3)
+ out += '{:.1f}'.format(rate)
+ else:
+ out += '?'
+ elif c == 'c':
+ self.current_rate.update_rate(self.finished_edges, self.time_millis)
+ if self.current_rate.rate == -1:
+ out += '?'
+ else:
+ out += '{:.1f}'.format(self.current_rate.rate)
+ elif c == 'p':
+ out += '{:3d}%'.format((100 * self.finished_edges) // self.total_edges)
+ elif c == 'e':
+ out += '{:.3f}'.format(self.time_millis / 1e3)
+ else:
+ raise RuntimeError('unknown placeholder '' + c +'' in $NINJA_STATUS')
+ else:
+ out += c
+ return out
+
+ def print_status(self, edge_started):
+ to_print = edge_started.desc
+ if self.verbose or to_print == '':
+ to_print = edge_started.command
+
+ to_print = self.format_progress_status(self.progress_status_format) + to_print
+
+ self.printer.print_line(to_print, LinePrinter.LINE_FULL if self.verbose else LinePrinter.LINE_ELIDE)
+
+
+def elide_middle(status, width):
+ margin = 3 # Space for '...'.
+ if len(status) + margin > width:
+ elide_size = (width - margin) // 2
+ status = status[0:elide_size] + '...' + status[-elide_size:]
+ return status
+
+class LinePrinter(object):
+ LINE_FULL = 1
+ LINE_ELIDE = 2
+
+ def __init__(self):
+ # Whether we can do fancy terminal control codes.
+ self.smart_terminal = False
+
+ # Whether the caret is at the beginning of a blank line.
+ self.have_blank_line = True
+
+ # Whether console is locked.
+ self.console_locked = False
+
+ # Buffered current line while console is locked.
+ self.line_buffer = ''
+
+ # Buffered line type while console is locked.
+ self.line_type = self.LINE_FULL
+
+ # Buffered console output while console is locked.
+ self.output_buffer = ''
+
+ if os.name == 'windows':
+ STD_OUTPUT_HANDLE = -11
+ self.console = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
+ csbi = ctypes.create_string_buffer(22)
+ self.smart_terminal = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi)
+ else:
+ term = os.getenv('TERM')
+ self.smart_terminal = os.isatty(1) and term != '' and term != 'dumb'
+
+ def print_line(self, to_print, line_type):
+ if self.console_locked:
+ self.line_buffer = to_print
+ self.line_type = line_type
+
+ if self.smart_terminal:
+ sys.stdout.write('\r') # Print over previous line, if any.
+
+ if self.smart_terminal and line_type == self.LINE_ELIDE:
+ if os.name == 'windows':
+ csbi = ctypes.create_string_buffer(22)
+ ctypes.windll.kernel32.GetConsoleScreenBufferInfo(self.console, csbi)
+ (cols, rows) = struct.unpack('hh', csbi.raw)
+ to_print = elide_middle(to_print, cols)
+ # TODO: windows support
+ # We don't want to have the cursor spamming back and forth, so instead of
+ # printf use WriteConsoleOutput which updates the contents of the buffer,
+ # but doesn't move the cursor position.
+ sys.stdout.write(to_print)
+ sys.stdout.flush()
+ else:
+ # Limit output to width of the terminal if provided so we don't cause
+ # line-wrapping.
+ import fcntl, termios
+ winsize = fcntl.ioctl(0, termios.TIOCGWINSZ, '\0'*4)
+ (rows, cols) = struct.unpack('hh', winsize)
+ to_print = elide_middle(to_print, cols)
+ sys.stdout.write(to_print)
+ sys.stdout.write('\x1B[K') # Clear to end of line.
+ sys.stdout.flush()
+
+ self.have_blank_line = False
+ else:
+ sys.stdout.write(to_print + '\n')
+ sys.stdout.flush()
+
+ def print_or_buffer(self, to_print):
+ if self.console_locked:
+ self.output_buffer += to_print
+ else:
+ sys.stdout.write(to_print)
+ sys.stdout.flush()
+
+ def print_on_new_line(self, to_print):
+ if self.console_locked or self.line_buffer != '':
+ self.output_buffer += self.line_buffer + '\n'
+ self.line_buffer = ''
+ if not self.have_blank_line:
+ self.print_or_buffer('\n')
+ if to_print != '':
+ self.print_or_buffer(to_print)
+ self.have_blank_line = to_print == '' or to_print[0] == '\n'
+
+ def set_console_locked(self, locked):
+ if locked == self.console_locked:
+ return
+
+ if locked:
+ self.print_on_new_line('\n')
+
+ self.console_locked = locked
+
+ if not locked:
+ self.print_on_new_line(self.output_buffer)
+ if self.line_buffer != '':
+ self.print_line(self.line_buffer, self.line_type)
+ self.output_buffer = ''
+ self.line_buffer = ''
+
+def main():
+ native = NinjaNativeFrontend()
+ for msg in frontend.Frontend():
+ native.handle(msg)
+
+if __name__ == '__main__':
+ main()
diff --git a/misc/generate_proto_header.py b/misc/generate_proto_header.py
new file mode 100755
index 0000000..a07eda6
--- /dev/null
+++ b/misc/generate_proto_header.py
@@ -0,0 +1,491 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write-only protobuf C++ code generator for a minimal runtime
+
+This script uses a descriptor proto generated by protoc and the descriptor_pb2
+distributed with python protobuf to iterate through the fields in a proto
+and write out simple C++ data objects with serialization methods. The generated
+files depend on a tiny runtime implemented in src/proto.h and src/proto.cc.
+"""
+
+from __future__ import print_function
+
+import os.path
+import re
+import sys
+
+import google.protobuf.descriptor_pb2
+import google.protobuf.descriptor
+
+FieldDescriptor = google.protobuf.descriptor.FieldDescriptor
+
+CPP_TYPE_MAP = {
+ FieldDescriptor.CPPTYPE_INT32: 'int32_t',
+ FieldDescriptor.CPPTYPE_INT64: 'int64_t',
+ FieldDescriptor.CPPTYPE_UINT32: 'uint32_t',
+ FieldDescriptor.CPPTYPE_UINT64: 'uint64_t',
+ FieldDescriptor.CPPTYPE_DOUBLE: 'double',
+ FieldDescriptor.CPPTYPE_FLOAT: 'float',
+ FieldDescriptor.CPPTYPE_BOOL: 'bool',
+ FieldDescriptor.CPPTYPE_STRING: 'std::string',
+}
+
+ENCODING_MAP = {
+ FieldDescriptor.TYPE_INT32:
+ ('VarintSize32SignExtended', 'WriteVarint32SignExtended', None),
+ FieldDescriptor.TYPE_INT64:
+ ('VarintSize64', 'WriteVarint64', None),
+ FieldDescriptor.TYPE_UINT32:
+ ('VarintSize32', 'WriteVarint32', None),
+ FieldDescriptor.TYPE_UINT64:
+ ('VarintSize64', 'WriteVarint64', None),
+ FieldDescriptor.TYPE_SINT32:
+ ('VarintSize32', 'WriteVarint32', 'ZigZagEncode32'),
+ FieldDescriptor.TYPE_SINT64:
+ ('VarintSize64', 'WriteVarint64', 'ZigZagEncode64'),
+ FieldDescriptor.TYPE_BOOL:
+ ('VarintSizeBool', 'WriteVarint32', None),
+ FieldDescriptor.TYPE_ENUM:
+ ('VarintSize32SignExtended', 'WriteVarint32SignExtended',
+ 'static_cast<int32_t>'),
+ FieldDescriptor.TYPE_FIXED64:
+ ('FixedSize64', 'WriteFixed64', None),
+ FieldDescriptor.TYPE_SFIXED64:
+ ('FixedSize64', 'WriteFixed64', 'static_cast<uint64_t>'),
+ FieldDescriptor.TYPE_DOUBLE:
+ ('FixedSize64', 'WriteFixed64', 'static_cast<uint64_t>'),
+ FieldDescriptor.TYPE_STRING:
+ ('StringSize', 'WriteString', None),
+ FieldDescriptor.TYPE_BYTES:
+ ('StringSize', 'WriteString', None),
+ FieldDescriptor.TYPE_FIXED32:
+ ('FixedSize32', 'WriteFixed32', None),
+ FieldDescriptor.TYPE_SFIXED32:
+ ('FixedSize32', 'WriteFixed32', 'static_cast<uint32_t>'),
+ FieldDescriptor.TYPE_FLOAT:
+ ('FixedSize32', 'WriteFixed32', 'static_cast<uint32_t>'),
+}
+
+ZIGZAG_LIST = (
+ FieldDescriptor.TYPE_SINT32,
+ FieldDescriptor.TYPE_SINT64,
+ FieldDescriptor.TYPE_SFIXED64,
+ FieldDescriptor.TYPE_SFIXED32,
+)
+
+def field_type_to_cpp_type(field):
+ """Convert a proto field object to its C++ type"""
+ if field.type_name != '':
+ cpp_type = field.type_name.replace('.', '::')
+ else:
+ cpp_type = FieldDescriptor.ProtoTypeToCppProtoType(field.type)
+ cpp_type = CPP_TYPE_MAP[cpp_type]
+ return cpp_type
+
+class Generator:
+ def __init__(self, out):
+ self.w = Writer(out)
+
+ """Class to generate C++ code for a proto descriptor"""
+ def write_enum(self, enum):
+ """Write a proto enum object to the generated file"""
+ self.w.writelines("""
+ enum %(name)s {
+ """ % {
+ 'name': enum.name,
+ })
+ self.w.indent()
+ for value in enum.value:
+ self.w.writelines("""
+ %(name)s = %(value)d,
+ """ % {
+ 'name': value.name,
+ 'value': value.number,
+ })
+ self.w.unindent()
+ self.w.writelines("""
+ };
+
+ """)
+
+ def write_field(self, field, ctor, ser, size, clear, methods):
+ """Write a proto field object to the generated file, including necessary
+ code in the constructor and serialization methods.
+ """
+ field_cpp_type = field_type_to_cpp_type(field)
+ repeated = field.label == FieldDescriptor.LABEL_REPEATED
+
+ element_cpp_type = field_cpp_type
+ if repeated:
+ field_cpp_type = 'std::vector< %s >' % field_cpp_type
+
+ member_name = field.name + '_'
+ element_name = member_name
+
+ # Data declaration
+ self.w.writelines("""
+ %(type)s %(member_name)s;
+ bool has_%(member_name)s;
+ """ % {
+ 'type': field_cpp_type,
+ 'member_name': member_name,
+ })
+
+ ctor.writelines("""
+ has_%(member_name)s = false;
+ """ % {
+ 'member_name': member_name,
+ })
+
+ methods.writelines("""
+ %(type)s* mutable_%(name)s() {
+ has_%(member_name)s = true;
+ return &%(member_name)s;
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ })
+
+ if repeated:
+ loop = """
+ for (%(type)s::const_iterator it_ = %(member_name)s.begin();
+ it_ != %(member_name)s.end(); it_++) {
+ """ % {
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ }
+
+ ser.writelines(loop)
+ ser.indent()
+
+ size.writelines(loop)
+ size.indent()
+
+ methods.writelines("""
+ void add_%(name)s(const %(type)s& value) {
+ has_%(member_name)s = true;
+ %(member_name)s.push_back(value);
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': element_cpp_type,
+ })
+
+ element_name = '*it_'
+
+ if field.type == FieldDescriptor.TYPE_MESSAGE:
+ ser.writelines("""
+ if (has_%(member_name)s) {
+ WriteLengthDelimited(output__, %(number)s,
+ %(member_name)s.ByteSizeLong());
+ %(member_name)s.SerializeToOstream(output__);
+ }
+ """ % {
+ 'member_name': element_name,
+ 'number': field.number,
+ })
+
+ size.writelines("""
+ if (has_%(member_name)s) {
+ size += 1 + VarintSize32(%(member_name)s.ByteSizeLong());
+ size += %(member_name)s.ByteSizeLong();
+ }
+ """ % {
+ 'member_name': element_name,
+ })
+
+ clear.writelines("""
+ if (has_%(member_name)s) {
+ %(member_name)s.Clear();
+ has_%(member_name)s = false;
+ }
+ """ % {
+ 'member_name': member_name,
+ })
+ else:
+ (sizer, serializer, formatter) = ENCODING_MAP[field.type]
+ if formatter != None:
+ element_name = '%s(%s)' % (formatter, element_name)
+
+ ser.writelines("""
+ %(serializer)s(output__, %(field_number)s, %(element_name)s);
+ """ % {
+ 'serializer': serializer,
+ 'field_number': field.number,
+ 'element_name': element_name,
+ })
+
+ size.writelines("""
+ size += %(sizer)s(%(element_name)s) + 1;
+ """ % {
+ 'sizer': sizer,
+ 'element_name': element_name,
+ })
+
+ if repeated or field.type == FieldDescriptor.CPPTYPE_STRING:
+ clear.writelines("""
+ %(member_name)s.clear();
+ """ % {
+ 'member_name': member_name,
+ })
+ else:
+ reset = """
+ %(member_name)s = static_cast< %(type)s >(0);
+ """ % {
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ }
+ ctor.writelines(reset)
+ clear.writelines(reset)
+
+ methods.writelines("""
+ void set_%(name)s(const %(type)s& value) {
+ has_%(member_name)s = true;
+ %(member_name)s = value;
+ }
+ """ % {
+ 'name': field.name,
+ 'member_name': member_name,
+ 'type': field_cpp_type,
+ })
+
+ if repeated:
+ ser.unindent()
+ ser.writelines('}')
+ size.unindent()
+ size.writelines('}')
+
+ def func(self, f):
+ return self.w.stringwriter(prefix=f + ' {', suffix='}\n\n')
+
+ def write_message(self, message):
+ """Write a proto message object to the generated file, recursing into
+ nested messages, enums, and fields.
+ """
+ self.w.writelines("""
+ struct %(name)s {
+ """ % {
+ 'name': message.name,
+ })
+ self.w.indent()
+
+ # Constructor
+ ctor = self.func(message.name + '()')
+
+ # SerializeToOstream method
+ ser = self.func('void SerializeToOstream(std::ostream* output__) const')
+
+ size = self.func('size_t ByteSizeLong() const')
+ size.writelines("""
+ size_t size = 0;
+ """)
+
+ clear = self.func('void Clear()')
+
+ methods = self.w.stringwriter()
+
+ # Nested message type declarations
+ for nested in message.nested_type:
+ self.write_message(nested)
+
+ # Nested enum type declarations
+ for enum in message.enum_type:
+ self.write_enum(enum)
+
+ # Message fields
+ for field in message.field:
+ self.write_field(field, ctor, ser, size, clear, methods)
+ if len(message.field) > 0:
+ self.w.newline()
+
+ self.w.writelines(ctor.string())
+
+ # Disallow copy and assign constructors
+ self.w.writelines("""
+ %(name)s(const %(name)s&);
+ void operator=(const %(name)s&);
+
+ """ % {
+ 'name': message.name,
+ })
+
+ # SerializeToOstream method
+ self.w.writelines(ser.string())
+
+ # ByteSizeLong method
+ size.writelines('return size;')
+ self.w.writelines(size.string())
+
+ # Clear method
+ self.w.writelines(clear.string())
+
+ # Accessor methods
+ self.w.write(methods.string())
+
+ self.w.unindent()
+ self.w.writelines("""
+ };
+
+ """)
+
+ def write_proto(self, output_file, proto):
+ header_guard = 'NINJA_' + os.path.basename(output_file).upper()
+ header_guard = re.sub('[^a-zA-Z]', '_', header_guard)
+
+ self.w.writelines("""
+ // This file is autogenerated by %(generator)s, do not edit
+
+ #ifndef %(header_guard)s
+ #define %(header_guard)s
+
+ #include <inttypes.h>
+
+ #include <iostream>
+ #include <string>
+ #include <vector>
+
+ #include "proto.h"
+
+ namespace %(namespace)s {
+ """ % {
+ 'generator': os.path.basename(sys.argv[0]),
+ 'header_guard': header_guard,
+ 'namespace': proto.package,
+ })
+
+ for enum in proto.enum_type:
+ self.write_enum(enum)
+ for message in proto.message_type:
+ self.write_message(message)
+
+ self.w.writelines("""
+ }
+ #endif // %(header_guard)s
+ """ % {
+ 'header_guard': header_guard,
+ })
+
+class Writer:
+ """Class to write code to a generated file"""
+ def __init__(self, w, indent=0):
+ self.w = w
+ self.cur_indent = indent
+
+ def write(self, s):
+ self.w.write(s)
+
+ def writeln(self, s):
+ if len(s) > 0:
+ self.write(' '*self.cur_indent + s + '\n')
+ else:
+ self.newline()
+
+ def indent(self):
+ self.cur_indent = self.cur_indent + 2
+
+ def unindent(self):
+ self.cur_indent = self.cur_indent - 2
+
+ def newline(self):
+ self.write('\n')
+
+ def writelines(self, s):
+ lines = s.split('\n')
+ if len(lines) > 0:
+ if len(lines[0].strip()) == 0:
+ lines = lines[1:]
+ if len(lines) > 0:
+ first_indent = initial_indent(lines[0])
+
+ for line in lines[:-1]:
+ indent = min(initial_indent(line), first_indent)
+ self.writeln(line[indent:])
+
+ indent = min(initial_indent(lines[-1]), first_indent)
+ if lines[-1][indent:] != '':
+ self.writeln(lines[-1][indent:])
+
+ def stringwriter(self, prefix='', suffix=''):
+ """Returns an object with the same interface as Writer that buffers
+ its writes to be written out later.
+ """
+ return StringWriter(self.cur_indent, prefix, suffix)
+
+def initial_indent(s):
+ return len(s)-len(s.lstrip(' '))
+
+class StringWriter(Writer):
+ """Subclass of Writer that buffers its writes to be written out later."""
+ def __init__(self, indent, prefix, suffix):
+ self.buf = ''
+ self.prefix = prefix
+ self.suffix = suffix
+ self.cur_indent = indent
+ if self.prefix != '':
+ self.writelines(self.prefix)
+ self.indent()
+
+ def string(self):
+ if self.prefix != '':
+ self.unindent()
+ if self.suffix != '':
+ self.writelines(self.suffix)
+ return self.buf
+
+ def write(self, s):
+ self.buf += s
+
+def main():
+ if len(sys.argv) == 2 and sys.argv[1] == '--probe':
+ print('ok')
+ return
+
+ if len(sys.argv) != 3:
+ print('usage: %s <in> <out>' % sys.argv[0])
+ sys.exit(1)
+
+ input_file = sys.argv[1]
+ output_file = sys.argv[2]
+ tmp_output_file = output_file + '.tmp'
+
+ set = google.protobuf.descriptor_pb2.FileDescriptorSet()
+ try:
+ with open(input_file, 'rb') as f:
+ set.ParseFromString(f.read())
+ except IOError:
+ print('failed to read ' + input_file)
+ sys.exit(2)
+
+ if len(set.file) != 1:
+ print('expected exactly one file descriptor in ' + input_file)
+ print(set)
+ sys.exit(3)
+
+ proto = set.file[0]
+
+ with open(tmp_output_file, 'w') as out:
+ w = Generator(out)
+
+ w.write_proto(output_file, proto)
+
+ os.rename(tmp_output_file, output_file)
+
+if __name__ == '__main__':
+ main()
diff --git a/misc/generate_proto_header_test.py b/misc/generate_proto_header_test.py
new file mode 100755
index 0000000..318e803
--- /dev/null
+++ b/misc/generate_proto_header_test.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write-only protobuf C++ code generator for a minimal runtime
+
+This script uses a descriptor proto generated by protoc and the descriptor_pb2
+distributed with python protobuf to iterate through the fields in a proto
+and write out simple C++ data objects with serialization methods. The generated
+files depend on a tiny runtime implemented in src/proto.h and src/proto.cc.
+"""
+
+from __future__ import print_function
+
+import unittest
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
+
+import generate_proto_header
+
+class TestWriteLines(unittest.TestCase):
+ def setUp(self):
+ self.out = StringIO()
+ self.w = generate_proto_header.Writer(self.out)
+
+ def test_single_line(self):
+ self.w.writelines('abc')
+ self.assertEqual(self.out.getvalue(), 'abc\n')
+
+ def test_multi_line(self):
+ self.w.writelines("""
+ abc
+ def
+ """)
+ self.assertEqual(self.out.getvalue(), 'abc\ndef\n')
+
+ def test_multi_line_with_indent(self):
+ self.w.writelines("""
+ abc
+ def
+ """)
+ self.assertEqual(self.out.getvalue(), 'abc\n def\n')
+
+ def test_string_writer(self):
+ w = self.w.stringwriter()
+ w.writelines("""
+ abc
+ def
+ """)
+ self.w.writelines(w.string())
+ self.assertEqual(self.out.getvalue(), 'abc\n def\n')
+
+ def test_string_writer_prefix(self):
+ w = self.w.stringwriter(
+ prefix="""
+ abc
+ def
+ """,
+ suffix="""
+ lmnop
+ """)
+ w.writelines("""
+ ghi
+ jk
+ """)
+ self.w.writelines(w.string())
+ self.assertEqual(self.out.getvalue(),
+ 'abc\n def\n ghi\n jk\nlmnop\n')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/src/build.cc b/src/build.cc
index b392803..2f8b860 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -36,7 +36,9 @@
#include "deps_log.h"
#include "disk_interface.h"
#include "graph.h"
+#include "metrics.h"
#include "state.h"
+#include "status.h"
#include "subprocess.h"
#include "util.h"
@@ -76,217 +78,6 @@
} // namespace
-BuildStatus::BuildStatus(const BuildConfig& config)
- : config_(config),
- start_time_millis_(GetTimeMillis()),
- started_edges_(0), finished_edges_(0), total_edges_(0),
- progress_status_format_(NULL),
- overall_rate_(), current_rate_(config.parallelism) {
-
- // Don't do anything fancy in verbose mode.
- if (config_.verbosity != BuildConfig::NORMAL)
- printer_.set_smart_terminal(false);
-
- progress_status_format_ = getenv("NINJA_STATUS");
- if (!progress_status_format_)
- progress_status_format_ = "[%f/%t] ";
-}
-
-void BuildStatus::PlanHasTotalEdges(int total) {
- total_edges_ = total;
-}
-
-void BuildStatus::BuildEdgeStarted(Edge* edge) {
- int start_time = (int)(GetTimeMillis() - start_time_millis_);
- running_edges_.insert(make_pair(edge, start_time));
- ++started_edges_;
-
- if (edge->use_console() || printer_.is_smart_terminal())
- PrintStatus(edge, kEdgeStarted);
-
- if (edge->use_console())
- printer_.SetConsoleLocked(true);
-}
-
-void BuildStatus::BuildEdgeFinished(Edge* edge,
- bool success,
- const string& output,
- int* start_time,
- int* end_time) {
- int64_t now = GetTimeMillis();
-
- ++finished_edges_;
-
- RunningEdgeMap::iterator i = running_edges_.find(edge);
- *start_time = i->second;
- *end_time = (int)(now - start_time_millis_);
- running_edges_.erase(i);
-
- if (edge->use_console())
- printer_.SetConsoleLocked(false);
-
- if (config_.verbosity == BuildConfig::QUIET)
- return;
-
- if (!edge->use_console())
- PrintStatus(edge, kEdgeFinished);
-
- // Print the command that is spewing before printing its output.
- if (!success) {
- string outputs;
- for (vector<Node*>::const_iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o)
- outputs += (*o)->path() + " ";
-
- printer_.PrintOnNewLine("FAILED: " + outputs + "\n");
- printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n");
- }
-
- if (!output.empty()) {
- // ninja sets stdout and stderr of subprocesses to a pipe, to be able to
- // check if the output is empty. Some compilers, e.g. clang, check
- // isatty(stderr) to decide if they should print colored output.
- // To make it possible to use colored output with ninja, subprocesses should
- // be run with a flag that forces them to always print color escape codes.
- // To make sure these escape codes don't show up in a file if ninja's output
- // is piped to a file, ninja strips ansi escape codes again if it's not
- // writing to a |smart_terminal_|.
- // (Launching subprocesses in pseudo ttys doesn't work because there are
- // only a few hundred available on some systems, and ninja can launch
- // thousands of parallel compile commands.)
- string final_output;
- if (!printer_.supports_color())
- final_output = StripAnsiEscapeCodes(output);
- else
- final_output = output;
-
-#ifdef _WIN32
- // Fix extra CR being added on Windows, writing out CR CR LF (#773)
- _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix
-#endif
-
- printer_.PrintOnNewLine(final_output);
-
-#ifdef _WIN32
- _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix
-#endif
- }
-}
-
-void BuildStatus::BuildStarted() {
- overall_rate_.Restart();
- current_rate_.Restart();
-}
-
-void BuildStatus::BuildFinished() {
- printer_.SetConsoleLocked(false);
- printer_.PrintOnNewLine("");
-}
-
-string BuildStatus::FormatProgressStatus(
- const char* progress_status_format, EdgeStatus status) const {
- string out;
- char buf[32];
- int percent;
- for (const char* s = progress_status_format; *s != '\0'; ++s) {
- if (*s == '%') {
- ++s;
- switch (*s) {
- case '%':
- out.push_back('%');
- break;
-
- // Started edges.
- case 's':
- snprintf(buf, sizeof(buf), "%d", started_edges_);
- out += buf;
- break;
-
- // Total edges.
- case 't':
- snprintf(buf, sizeof(buf), "%d", total_edges_);
- out += buf;
- break;
-
- // Running edges.
- case 'r': {
- int running_edges = started_edges_ - finished_edges_;
- // count the edge that just finished as a running edge
- if (status == kEdgeFinished)
- running_edges++;
- snprintf(buf, sizeof(buf), "%d", running_edges);
- out += buf;
- break;
- }
-
- // Unstarted edges.
- case 'u':
- snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_);
- out += buf;
- break;
-
- // Finished edges.
- case 'f':
- snprintf(buf, sizeof(buf), "%d", finished_edges_);
- out += buf;
- break;
-
- // Overall finished edges per second.
- case 'o':
- overall_rate_.UpdateRate(finished_edges_);
- SnprintfRate(overall_rate_.rate(), buf, "%.1f");
- out += buf;
- break;
-
- // Current rate, average over the last '-j' jobs.
- case 'c':
- current_rate_.UpdateRate(finished_edges_);
- SnprintfRate(current_rate_.rate(), buf, "%.1f");
- out += buf;
- break;
-
- // Percentage
- case 'p':
- percent = (100 * finished_edges_) / total_edges_;
- snprintf(buf, sizeof(buf), "%3i%%", percent);
- out += buf;
- break;
-
- case 'e': {
- double elapsed = overall_rate_.Elapsed();
- snprintf(buf, sizeof(buf), "%.3f", elapsed);
- out += buf;
- break;
- }
-
- default:
- Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
- return "";
- }
- } else {
- out.push_back(*s);
- }
- }
-
- return out;
-}
-
-void BuildStatus::PrintStatus(Edge* edge, EdgeStatus status) {
- if (config_.verbosity == BuildConfig::QUIET)
- return;
-
- bool force_full_command = config_.verbosity == BuildConfig::VERBOSE;
-
- string to_print = edge->GetBinding("description");
- if (to_print.empty() || force_full_command)
- to_print = edge->GetBinding("command");
-
- to_print = FormatProgressStatus(progress_status_format_, status) + to_print;
-
- printer_.Print(to_print,
- force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE);
-}
-
Plan::Plan() : command_edges_(0), wanted_edges_(0) {}
void Plan::Reset() {
@@ -348,7 +139,7 @@
Edge* Plan::FindWork() {
if (ready_.empty())
return NULL;
- set<Edge*>::iterator e = ready_.begin();
+ EdgeSet::iterator e = ready_.begin();
Edge* edge = *e;
ready_.erase(e);
return edge;
@@ -404,8 +195,9 @@
void Plan::NodeFinished(Node* node) {
// See if we we want any edges from this node.
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end())
continue;
@@ -426,8 +218,9 @@
bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
node->set_dirty(false);
- for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
- oe != node->out_edges().end(); ++oe) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator oe = out_edges.begin();
+ oe != out_edges.end(); ++oe) {
// Don't process edges that we don't actually want.
map<Edge*, Want>::iterator want_e = want_.find(*oe);
if (want_e == want_.end() || want_e->second == kWantNothing)
@@ -437,6 +230,10 @@
if ((*oe)->deps_missing_)
continue;
+ // No need to clean a phony output edge, as it's always dirty
+ if ((*oe)->IsPhonyOutput())
+ continue;
+
// If all non-order-only inputs for this edge are now clean,
// we might have changed the dirty state of the outputs.
vector<Node*>::iterator
@@ -543,6 +340,9 @@
}
result->status = subproc->Finish();
+#ifndef _WIN32
+ result->rusage = *subproc->GetUsage();
+#endif
result->output = subproc->GetOutput();
map<Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc);
@@ -555,11 +355,12 @@
Builder::Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
- DiskInterface* disk_interface)
- : state_(state), config_(config), disk_interface_(disk_interface),
+ DiskInterface* disk_interface, Status* status,
+ int64_t start_time_millis)
+ : state_(state), config_(config), status_(status),
+ start_time_millis_(start_time_millis), disk_interface_(disk_interface),
scan_(state, build_log, deps_log, disk_interface,
- &config_.depfile_parser_options) {
- status_ = new BuildStatus(config);
+ &config_.depfile_parser_options, config.uses_phony_outputs) {
}
Builder::~Builder() {
@@ -573,6 +374,8 @@
for (vector<Edge*>::iterator e = active_edges.begin();
e != active_edges.end(); ++e) {
+ if ((*e)->IsPhonyOutput())
+ continue;
string depfile = (*e)->GetUnescapedDepfile();
for (vector<Node*>::iterator o = (*e)->outputs_.begin();
o != (*e)->outputs_.end(); ++o) {
@@ -584,10 +387,11 @@
// mentioned in a depfile, and the command touches its depfile
// but is interrupted before it touches its output file.)
string err;
- TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err);
- if (new_mtime == -1) // Log and ignore Stat() errors.
- Error("%s", err.c_str());
- if (!depfile.empty() || (*o)->mtime() != new_mtime)
+ bool is_dir = false;
+ TimeStamp new_mtime = disk_interface_->LStat((*o)->path(), &is_dir, &err);
+ if (new_mtime == -1) // Log and ignore LStat() errors.
+ status_->Error("%s", err.c_str());
+ if (!is_dir && (!depfile.empty() || (*o)->mtime() != new_mtime))
disk_interface_->RemoveFile((*o)->path());
}
if (!depfile.empty())
@@ -602,22 +406,41 @@
*err = "unknown target: '" + name + "'";
return NULL;
}
- if (!AddTarget(node, err))
+ if (!AddTargets({ node }, err))
return NULL;
return node;
}
-bool Builder::AddTarget(Node* node, string* err) {
- if (!scan_.RecomputeDirty(node, err))
+bool Builder::AddTargets(const std::vector<Node*> &nodes, string* err) {
+ std::vector<Node*> validation_nodes;
+ if (!scan_.RecomputeNodesDirty(nodes, &validation_nodes, err))
return false;
- if (Edge* in_edge = node->in_edge()) {
- if (in_edge->outputs_ready())
- return true; // Nothing to do.
+ for (Node* node : nodes) {
+ std::string plan_err;
+ if (!plan_.AddTarget(node, &plan_err)) {
+ if (!plan_err.empty()) {
+ *err = plan_err;
+ return false;
+ } else {
+ // Added a target that is already up-to-date; not really
+ // an error.
+ }
+ }
}
- if (!plan_.AddTarget(node, err))
- return false;
+ for (Node* node : validation_nodes) {
+ std::string plan_err;
+ if (!plan_.AddTarget(node, &plan_err)) {
+ if (!plan_err.empty()) {
+ *err = plan_err;
+ return false;
+ } else {
+ // Added a target that is already up-to-date; not really
+ // an error.
+ }
+ }
+ }
return true;
}
@@ -721,14 +544,29 @@
if (edge->is_phony())
return true;
- status_->BuildEdgeStarted(edge);
+ int64_t start_time_millis = GetTimeMillis() - start_time_millis_;
+ running_edges_.insert(make_pair(edge, start_time_millis));
- // Create directories necessary for outputs.
- // XXX: this will block; do we care?
- for (vector<Node*>::iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o) {
- if (!disk_interface_->MakeDirs((*o)->path()))
- return false;
+ status_->BuildEdgeStarted(edge, start_time_millis);
+
+ if (!edge->IsPhonyOutput()) {
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ // Create directories necessary for outputs.
+ // XXX: this will block; do we care?
+ if (!disk_interface_->MakeDirs((*o)->path()))
+ return false;
+
+ if (!(*o)->exists())
+ continue;
+
+ // Remove existing outputs for non-restat rules.
+ // XXX: this will block; do we care?
+ if (config_.pre_remove_output_files && !edge->IsRestat() && !config_.dry_run) {
+ if (disk_interface_->RemoveFile((*o)->path()) < 0)
+ return false;
+ }
+ }
}
// Create response file, if needed
@@ -753,72 +591,115 @@
METRIC_RECORD("FinishCommand");
Edge* edge = result->edge;
+ bool phony_output = edge->IsPhonyOutput();
- // First try to extract dependencies from the result, if any.
- // This must happen first as it filters the command output (we want
- // to filter /showIncludes output, even on compile failure) and
- // extraction itself can fail, which makes the command fail from a
- // build perspective.
vector<Node*> deps_nodes;
string deps_type = edge->GetBinding("deps");
- const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
- if (!deps_type.empty()) {
- string extract_err;
- if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
- &extract_err) &&
- result->success()) {
- if (!result->output.empty())
- result->output.append("\n");
- result->output.append(extract_err);
- result->status = ExitFailure;
+ if (!phony_output) {
+ // First try to extract dependencies from the result, if any.
+ // This must happen first as it filters the command output (we want
+ // to filter /showIncludes output, even on compile failure) and
+ // extraction itself can fail, which makes the command fail from a
+ // build perspective.
+ const string deps_prefix = edge->GetBinding("msvc_deps_prefix");
+ if (!deps_type.empty()) {
+ string extract_err;
+ if (!ExtractDeps(result, deps_type, deps_prefix, &deps_nodes,
+ &extract_err) &&
+ result->success()) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append(extract_err);
+ result->status = ExitFailure;
+ }
}
}
- int start_time, end_time;
- status_->BuildEdgeFinished(edge, result->success(), result->output,
- &start_time, &end_time);
-
- // The rest of this function only applies to successful commands.
- if (!result->success()) {
- plan_.EdgeFinished(edge, Plan::kEdgeFailed);
- return true;
- }
+ int64_t start_time_millis, end_time_millis;
+ RunningEdgeMap::iterator i = running_edges_.find(edge);
+ start_time_millis = i->second;
+ end_time_millis = GetTimeMillis() - start_time_millis_;
+ running_edges_.erase(i);
// Restat the edge outputs
TimeStamp output_mtime = 0;
- bool restat = edge->GetBindingBool("restat");
- if (!config_.dry_run) {
- bool node_cleaned = false;
+ if (result->success() && !config_.dry_run && !phony_output) {
+ bool restat = edge->IsRestat();
+ vector<Node*> nodes_cleaned;
+
+ TimeStamp newest_input = 0;
+ Node* newest_input_node = nullptr;
+ for (vector<Node*>::iterator i = edge->inputs_.begin();
+ i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
+ TimeStamp input_mtime = (*i)->mtime();
+ if (input_mtime == -1)
+ return false;
+ if (input_mtime > newest_input) {
+ newest_input = input_mtime;
+ newest_input_node = (*i);
+ }
+ }
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
- TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err);
- if (new_mtime == -1)
+ bool is_dir = false;
+ TimeStamp old_mtime = (*o)->mtime();
+ if (!(*o)->LStat(disk_interface_, &is_dir, err))
return false;
+ TimeStamp new_mtime = (*o)->mtime();
+ if (config_.uses_phony_outputs) {
+ if (new_mtime == 0) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: output file missing after successful execution: ");
+ result->output.append((*o)->path());
+ if (config_.missing_output_file_should_err) {
+ result->status = ExitFailure;
+ }
+ } else if (!restat && new_mtime < newest_input) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: Missing `restat`? An output file is older than the most recent input:\n output: ");
+ result->output.append((*o)->path());
+ result->output.append("\n input: ");
+ result->output.append(newest_input_node->path());
+ if (config_.old_output_should_err) {
+ result->status = ExitFailure;
+ }
+ }
+ if (is_dir) {
+ if (!result->output.empty())
+ result->output.append("\n");
+ result->output.append("ninja: outputs should be files, not directories: ");
+ result->output.append((*o)->path());
+ if (config_.output_directory_should_err) {
+ result->status = ExitFailure;
+ }
+ }
+ }
if (new_mtime > output_mtime)
output_mtime = new_mtime;
- if ((*o)->mtime() == new_mtime && restat) {
+ if (old_mtime == new_mtime && restat) {
+ nodes_cleaned.push_back(*o);
+ continue;
+ }
+ }
+
+ status_->BuildEdgeFinished(edge, end_time_millis, result);
+
+ if (result->success() && !nodes_cleaned.empty()) {
+ for (vector<Node*>::iterator o = nodes_cleaned.begin();
+ o != nodes_cleaned.end(); ++o) {
// The rule command did not change the output. Propagate the clean
// state through the build graph.
// Note that this also applies to nonexistent outputs (mtime == 0).
if (!plan_.CleanNode(&scan_, *o, err))
return false;
- node_cleaned = true;
}
- }
- if (node_cleaned) {
- TimeStamp restat_mtime = 0;
// If any output was cleaned, find the most recent mtime of any
// (existing) non-order-only input or the depfile.
- for (vector<Node*>::iterator i = edge->inputs_.begin();
- i != edge->inputs_.end() - edge->order_only_deps_; ++i) {
- TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err);
- if (input_mtime == -1)
- return false;
- if (input_mtime > restat_mtime)
- restat_mtime = input_mtime;
- }
+ TimeStamp restat_mtime = newest_input;
string depfile = edge->GetUnescapedDepfile();
if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) {
@@ -835,27 +716,33 @@
output_mtime = restat_mtime;
}
+ } else {
+ status_->BuildEdgeFinished(edge, end_time_millis, result);
}
- plan_.EdgeFinished(edge, Plan::kEdgeSucceeded);
+ plan_.EdgeFinished(edge, result->success() ? Plan::kEdgeSucceeded : Plan::kEdgeFailed);
+
+ // The rest of this function only applies to successful commands.
+ if (!result->success()) {
+ return true;
+ }
// Delete any left over response file.
string rspfile = edge->GetUnescapedRspfile();
if (!rspfile.empty() && !g_keep_rsp)
disk_interface_->RemoveFile(rspfile);
- if (scan_.build_log()) {
- if (!scan_.build_log()->RecordCommand(edge, start_time, end_time,
- output_mtime)) {
+ if (scan_.build_log() && !phony_output) {
+ if (!scan_.build_log()->RecordCommand(edge, start_time_millis,
+ end_time_millis, output_mtime)) {
*err = string("Error writing to build log: ") + strerror(errno);
return false;
}
}
- if (!deps_type.empty() && !config_.dry_run) {
- assert(edge->outputs_.size() == 1 && "should have been rejected by parser");
+ if (!deps_type.empty() && !config_.dry_run && !phony_output) {
Node* out = edge->outputs_[0];
- TimeStamp deps_mtime = disk_interface_->Stat(out->path(), err);
+ TimeStamp deps_mtime = disk_interface_->LStat(out->path(), nullptr, err);
if (deps_mtime == -1)
return false;
if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) {
@@ -900,6 +787,15 @@
break;
case DiskInterface::NotFound:
err->clear();
+ // We only care if the depfile is missing when the tool succeeded.
+ if (!config_.dry_run && result->status == ExitSuccess) {
+ if (config_.missing_depfile_should_err) {
+ *err = string("depfile is missing");
+ return false;
+ } else {
+ status_->Warning("depfile is missing (%s for %s)", depfile.c_str(), result->edge->outputs_[0]->path().c_str());
+ }
+ }
break;
case DiskInterface::OtherError:
return false;
diff --git a/src/build.h b/src/build.h
index a42b8d4..5c06f8b 100644
--- a/src/build.h
+++ b/src/build.h
@@ -19,23 +19,24 @@
#include <map>
#include <memory>
#include <queue>
-#include <set>
#include <string>
#include <vector>
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
#include "depfile_parser.h"
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "exit_status.h"
-#include "line_printer.h"
-#include "metrics.h"
#include "util.h" // int64_t
struct BuildLog;
-struct BuildStatus;
struct DiskInterface;
struct Edge;
struct Node;
struct State;
+struct Status;
/// Plan stores the state of a build plan: what we intend to build,
/// which steps we're ready to execute.
@@ -103,7 +104,7 @@
/// we want for the edge.
map<Edge*, Want> want_;
- set<Edge*> ready_;
+ EdgeSet ready_;
/// Total number of edges that have commands (not phony).
int command_edges_;
@@ -125,6 +126,9 @@
Result() : edge(NULL) {}
Edge* edge;
ExitStatus status;
+#ifndef _WIN32
+ struct rusage rusage;
+#endif
string output;
bool success() const { return status == ExitSuccess; }
};
@@ -138,7 +142,14 @@
/// Options (e.g. verbosity, parallelism) passed to a build.
struct BuildConfig {
BuildConfig() : verbosity(NORMAL), dry_run(false), parallelism(1),
- failures_allowed(1), max_load_average(-0.0f) {}
+ failures_allowed(1), max_load_average(-0.0f),
+ frontend(NULL), frontend_file(NULL),
+ missing_depfile_should_err(false),
+ uses_phony_outputs(false),
+ output_directory_should_err(false),
+ missing_output_file_should_err(false),
+ old_output_should_err(false),
+ pre_remove_output_files(false) {}
enum Verbosity {
NORMAL,
@@ -153,23 +164,51 @@
/// means that we do not have any limit.
double max_load_average;
DepfileParserOptions depfile_parser_options;
+
+ /// Command to execute to handle build output
+ const char* frontend;
+
+ /// File to write build output to
+ const char* frontend_file;
+
+ /// Whether a missing depfile should warn or print an error.
+ bool missing_depfile_should_err;
+
+ /// Whether the generator uses 'phony_output's
+ /// Controls the warnings below
+ bool uses_phony_outputs;
+
+ /// Whether an output can be a directory
+ bool output_directory_should_err;
+
+ /// Whether a missing output file should warn or print an error.
+ bool missing_output_file_should_err;
+
+ /// Whether an output with an older timestamp than the inputs should
+ /// warn or print an error.
+ bool old_output_should_err;
+
+ /// Whether to remove outputs before executing rule commands
+ bool pre_remove_output_files;
};
/// Builder wraps the build process: starting commands, updating status.
struct Builder {
Builder(State* state, const BuildConfig& config,
BuildLog* build_log, DepsLog* deps_log,
- DiskInterface* disk_interface);
+ DiskInterface* disk_interface, Status* status,
+ int64_t start_time_millis);
~Builder();
/// Clean up after interrupted commands by deleting output files.
void Cleanup();
+ /// Used by tests.
Node* AddTarget(const string& name, string* err);
- /// Add a target to the build, scanning dependencies.
+ /// Add targets to the build, scanning dependencies.
/// @return false on error.
- bool AddTarget(Node* target, string* err);
+ bool AddTargets(const std::vector<Node*>& targets, string* err);
/// Returns true if the build targets are already up to date.
bool AlreadyUpToDate() const;
@@ -197,13 +236,20 @@
#else
unique_ptr<CommandRunner> command_runner_; // auto_ptr was removed in C++17.
#endif
- BuildStatus* status_;
+ Status* status_;
private:
bool ExtractDeps(CommandRunner::Result* result, const string& deps_type,
const string& deps_prefix, vector<Node*>* deps_nodes,
string* err);
+ /// Map of running edge to time the edge started running.
+ typedef map<Edge*, int> RunningEdgeMap;
+ RunningEdgeMap running_edges_;
+
+ /// Time the build started.
+ int64_t start_time_millis_;
+
DiskInterface* disk_interface_;
DependencyScan scan_;
@@ -212,102 +258,4 @@
void operator=(const Builder &other); // DO NOT IMPLEMENT
};
-/// Tracks the status of a build: completion fraction, printing updates.
-struct BuildStatus {
- explicit BuildStatus(const BuildConfig& config);
- void PlanHasTotalEdges(int total);
- void BuildEdgeStarted(Edge* edge);
- void BuildEdgeFinished(Edge* edge, bool success, const string& output,
- int* start_time, int* end_time);
- void BuildStarted();
- void BuildFinished();
-
- enum EdgeStatus {
- kEdgeStarted,
- kEdgeFinished,
- };
-
- /// Format the progress status string by replacing the placeholders.
- /// See the user manual for more information about the available
- /// placeholders.
- /// @param progress_status_format The format of the progress status.
- /// @param status The status of the edge.
- string FormatProgressStatus(const char* progress_status_format,
- EdgeStatus status) const;
-
- private:
- void PrintStatus(Edge* edge, EdgeStatus status);
-
- const BuildConfig& config_;
-
- /// Time the build started.
- int64_t start_time_millis_;
-
- int started_edges_, finished_edges_, total_edges_;
-
- /// Map of running edge to time the edge started running.
- typedef map<Edge*, int> RunningEdgeMap;
- RunningEdgeMap running_edges_;
-
- /// Prints progress output.
- LinePrinter printer_;
-
- /// The custom progress status format to use.
- const char* progress_status_format_;
-
- template<size_t S>
- void SnprintfRate(double rate, char(&buf)[S], const char* format) const {
- if (rate == -1)
- snprintf(buf, S, "?");
- else
- snprintf(buf, S, format, rate);
- }
-
- struct RateInfo {
- RateInfo() : rate_(-1) {}
-
- void Restart() { stopwatch_.Restart(); }
- double Elapsed() const { return stopwatch_.Elapsed(); }
- double rate() { return rate_; }
-
- void UpdateRate(int edges) {
- if (edges && stopwatch_.Elapsed())
- rate_ = edges / stopwatch_.Elapsed();
- }
-
- private:
- double rate_;
- Stopwatch stopwatch_;
- };
-
- struct SlidingRateInfo {
- SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
-
- void Restart() { stopwatch_.Restart(); }
- double rate() { return rate_; }
-
- void UpdateRate(int update_hint) {
- if (update_hint == last_update_)
- return;
- last_update_ = update_hint;
-
- if (times_.size() == N)
- times_.pop();
- times_.push(stopwatch_.Elapsed());
- if (times_.back() != times_.front())
- rate_ = times_.size() / (times_.back() - times_.front());
- }
-
- private:
- double rate_;
- Stopwatch stopwatch_;
- const size_t N;
- queue<double> times_;
- int last_update_;
- };
-
- mutable RateInfo overall_rate_;
- mutable SlidingRateInfo current_rate_;
-};
-
#endif // NINJA_BUILD_H_
diff --git a/src/build_log.cc b/src/build_log.cc
index c4a08a0..0ebc45d 100644
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -22,6 +22,7 @@
#include "build_log.h"
+#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
@@ -31,9 +32,13 @@
#include <unistd.h>
#endif
+#include <numeric>
+
#include "build.h"
+#include "disk_interface.h"
#include "graph.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "util.h"
#if defined(_MSC_VER) && (_MSC_VER < 1800)
#define strtoll _strtoi64
@@ -49,9 +54,11 @@
namespace {
const char kFileSignature[] = "# ninja log v%d\n";
-const int kOldestSupportedVersion = 4;
+const int kOldestSupportedVersion = 5;
const int kCurrentVersion = 5;
+const char kFieldSeparator = '\t';
+
// 64bit MurmurHash2, by Austin Appleby
#if defined(_MSC_VER)
#define BIG_CONSTANT(x) (x)
@@ -105,13 +112,14 @@
// static
uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
- return MurmurHash64A(command.str_, command.len_);
+ METRIC_RECORD("hash command");
+ return MurmurHash64A(command.data(), command.size());
}
-BuildLog::LogEntry::LogEntry(const string& output)
+BuildLog::LogEntry::LogEntry(const HashedStrView& output)
: output(output) {}
-BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
+BuildLog::LogEntry::LogEntry(const HashedStrView& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp restat_mtime)
: output(output), command_hash(command_hash),
start_time(start_time), end_time(end_time), mtime(restat_mtime)
@@ -157,13 +165,11 @@
TimeStamp mtime) {
string command = edge->EvaluateCommand(true);
uint64_t command_hash = LogEntry::HashCommand(command);
- for (vector<Node*>::iterator out = edge->outputs_.begin();
- out != edge->outputs_.end(); ++out) {
- const string& path = (*out)->path();
- Entries::iterator i = entries_.find(path);
+ for (Node* out : edge->outputs_) {
+ const HashedStr& path = out->path_hashed();
LogEntry* log_entry;
- if (i != entries_.end()) {
- log_entry = i->second;
+ if (LogEntry** i = entries_.Lookup(path)) {
+ log_entry = *i;
} else {
log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
@@ -190,164 +196,276 @@
log_file_ = NULL;
}
-struct LineReader {
- explicit LineReader(FILE* file)
- : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
- memset(buf_, 0, sizeof(buf_));
+/// Retrieve the next tab-delimited or LF-delimited piece in the input.
+///
+/// Specifically, the function searches for a separator, starting at the given
+/// input offset. If the function finds the separator, it removes everything up
+/// to and including the separator from |*input|, places it in |*out|, and
+/// returns true. Otherwise, it zeroes |*out| and returns false.
+static bool GetNextPiece(StringPiece* input, char sep, StringPiece* out,
+ size_t start_offset=0) {
+ assert(input != out);
+ const char* data = input->data();
+ const char* split = nullptr;
+
+ // memchr(NULL, NULL, 0) has undefined behavior, so avoid calling memchr when
+ // input->data() is nullptr. If input->size() is non-zero, its data() will be
+ // non-null.
+ if (start_offset < input->size()) {
+ split = static_cast<const char*>(
+ memchr(data + start_offset, sep, input->size() - start_offset));
}
- // Reads a \n-terminated line from the file passed to the constructor.
- // On return, *line_start points to the beginning of the next line, and
- // *line_end points to the \n at the end of the line. If no newline is seen
- // in a fixed buffer size, *line_end is set to NULL. Returns false on EOF.
- bool ReadLine(char** line_start, char** line_end) {
- if (line_start_ >= buf_end_ || !line_end_) {
- // Buffer empty, refill.
- size_t size_read = fread(buf_, 1, sizeof(buf_), file_);
- if (!size_read)
- return false;
- line_start_ = buf_;
- buf_end_ = buf_ + size_read;
- } else {
- // Advance to next line in buffer.
- line_start_ = line_end_ + 1;
- }
-
- line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
- if (!line_end_) {
- // No newline. Move rest of data to start of buffer, fill rest.
- size_t already_consumed = line_start_ - buf_;
- size_t size_rest = (buf_end_ - buf_) - already_consumed;
- memmove(buf_, line_start_, size_rest);
-
- size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_);
- buf_end_ = buf_ + size_rest + read;
- line_start_ = buf_;
- line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_);
- }
-
- *line_start = line_start_;
- *line_end = line_end_;
+ if (split != nullptr) {
+ size_t len = split + 1 - data;
+ *out = input->substr(0, len);
+ *input = input->substr(len);
return true;
+ } else {
+ *out = {};
+ return false;
}
+}
- private:
- FILE* file_;
- char buf_[256 << 10];
- char* buf_end_; // Points one past the last valid byte in |buf_|.
+struct BuildLogInput {
+ std::unique_ptr<LoadedFile> file;
+ int log_version = 0;
- char* line_start_;
- // Points at the next \n in buf_ after line_start, or NULL.
- char* line_end_;
+ // Content excluding the file header.
+ StringPiece content;
};
-bool BuildLog::Load(const string& path, string* err) {
- METRIC_RECORD(".ninja_log load");
- FILE* file = fopen(path.c_str(), "r");
- if (!file) {
- if (errno == ENOENT)
- return true;
- *err = strerror(errno);
+static bool OpenBuildLogForReading(const std::string& path,
+ BuildLogInput* log,
+ std::string* err) {
+ *log = {};
+
+ RealDiskInterface file_reader;
+ std::string load_err;
+ switch (file_reader.LoadFile(path, &log->file, &load_err)) {
+ case FileReader::Okay:
+ break;
+ case FileReader::NotFound:
+ return true;
+ default:
+ *err = load_err;
return false;
}
- int log_version = 0;
- int unique_entry_count = 0;
- int total_entry_count = 0;
+ // We need a NUL terminator after the log file's content so that we can call
+ // atoi/atol/strtoull with a pointer to within the content.
+ log->content = log->file->content_with_nul();
+ log->content.remove_suffix(1);
- LineReader reader(file);
- char* line_start = 0;
- char* line_end = 0;
- while (reader.ReadLine(&line_start, &line_end)) {
- if (!log_version) {
- sscanf(line_start, kFileSignature, &log_version);
+ StringPiece header_line;
+ if (GetNextPiece(&log->content, '\n', &header_line)) {
+ // At least with glibc, sscanf will touch every byte of the string it scans,
+ // so make a copy of the first line for sscanf to use. (Maybe sscanf is
+ // calling strlen internally?)
+ sscanf((header_line.AsString()).c_str(), kFileSignature,
+ &log->log_version);
+ }
- if (log_version < kOldestSupportedVersion) {
- *err = ("build log version invalid, perhaps due to being too old; "
- "starting over");
- fclose(file);
- unlink(path.c_str());
- // Don't report this as a failure. An empty build log will cause
- // us to rebuild the outputs anyway.
- return true;
+ if (log->log_version < kOldestSupportedVersion) {
+ *err = ("build log version invalid, perhaps due to being too old; "
+ "starting over");
+ log->content = {};
+ log->file.reset();
+ unlink(path.c_str());
+ // Don't report this as a failure. An empty build log will cause
+ // us to rebuild the outputs anyway.
+ }
+
+ return true;
+}
+
+/// Split the build log's content (i.e. the lines excluding the header) into
+/// chunks. Each chunk is guaranteed to end with a newline character. Any output
+/// beyond the end of the last newline is quietly discarded.
+static std::vector<StringPiece> SplitBuildLog(StringPiece content) {
+ // The log file should end with an LF character, but if it doesn't, start by
+ // stripping off non-LF characters.
+ while (!content.empty() && content.back() != '\n') {
+ content.remove_suffix(1);
+ }
+
+ size_t ideal_chunk_count = GetOptimalThreadPoolJobCount();
+ size_t ideal_chunk_size = content.size() / ideal_chunk_count + 1;
+
+ std::vector<StringPiece> result;
+ StringPiece chunk;
+ while (GetNextPiece(&content, '\n', &chunk, ideal_chunk_size)) {
+ result.push_back(chunk);
+ }
+ if (!content.empty()) {
+ result.push_back(content);
+ }
+
+ return result;
+}
+
+/// Call the given function on each line in the given chunk. The chunk must end
+/// with an LF character. The LF characters are included in the string views
+/// passed to the callback.
+template <typename Func>
+static void VisitEachLineInChunk(StringPiece chunk, Func func) {
+ assert(!chunk.empty() && chunk.back() == '\n');
+ StringPiece line;
+ while (GetNextPiece(&chunk, '\n', &line)) {
+ func(line);
+ }
+}
+
+/// Count the number of LF newline characters in the string. The string is
+/// guaranteed to end with an LF.
+static size_t CountNewlinesInChunk(StringPiece chunk) {
+ size_t line_count = 0;
+ VisitEachLineInChunk(chunk, [&line_count](StringPiece line) {
+ ++line_count;
+ });
+ return line_count;
+}
+
+struct ParsedLine {
+ /// These fields are guaranteed to be followed by a whitespace character
+ /// (either a tab or an LF), but the whitespace terminator isn't explicitly
+ /// part of the string piece.
+ StringPiece start_time;
+ StringPiece end_time;
+ StringPiece mtime;
+ StringPiece path;
+ StringPiece command_hash;
+};
+
+/// Given a single line of the build log (including the terminator LF), split
+/// the line into its various tab-delimited fields.
+static inline bool SplitLine(StringPiece line, ParsedLine* out) {
+ assert(!line.empty() && line.back() == '\n');
+ line.remove_suffix(1);
+
+ auto get_next_field = [&line](StringPiece* out_piece) {
+ // Extract the next piece from the line. If we're successful, remove the
+ // field separator from the end of the piece.
+ if (!GetNextPiece(&line, kFieldSeparator, out_piece)) return false;
+ assert(!out_piece->empty() && out_piece->back() == kFieldSeparator);
+ out_piece->remove_suffix(1);
+ return true;
+ };
+
+ *out = {};
+ if (!get_next_field(&out->start_time)) return false;
+ if (!get_next_field(&out->end_time)) return false;
+ if (!get_next_field(&out->mtime)) return false;
+ if (!get_next_field(&out->path)) return false;
+ out->command_hash = line;
+
+ return true;
+}
+
+/// Given a single line of the build log (including the terminator LF), return
+/// just the path field.
+static StringPiece GetPathForLine(StringPiece line) {
+ ParsedLine parsed_line;
+ return SplitLine(line, &parsed_line) ? parsed_line.path : StringPiece();
+}
+
+bool BuildLog::Load(const string& path, string* err) {
+ METRIC_RECORD(".ninja_log load");
+ assert(entries_.empty());
+
+ BuildLogInput log;
+ if (!OpenBuildLogForReading(path, &log, err)) return false;
+ if (log.file.get() == nullptr) return true;
+
+ std::vector<StringPiece> chunks = SplitBuildLog(log.content);
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ // The concurrent hashmap doesn't automatically resize when entries are added
+ // to it, so we need to reserve space in advance. The number of newlines
+ // should exceed the number of distinct paths in the build log by a small
+ // factor.
+ size_t line_count = 0;
+ for (size_t count :
+ ParallelMap(thread_pool.get(), chunks, CountNewlinesInChunk)) {
+ line_count += count;
+ }
+ entries_.reserve(line_count);
+
+ // Construct an initial table of path -> LogEntry. Each LogEntry is
+ // initialized with the newest record for a particular path. Build a list of
+ // each chunk's new log entries.
+ std::vector<std::vector<LogEntry*>> chunk_new_entries =
+ ParallelMap(thread_pool.get(), chunks, [this, &log](StringPiece chunk) {
+ std::vector<LogEntry*> chunk_entries_list;
+ VisitEachLineInChunk(chunk, [&](StringPiece line) {
+ HashedStrView path = GetPathForLine(line);
+ if (path.empty()) return;
+ LogEntry* entry = nullptr;
+ if (LogEntry** entry_it = entries_.Lookup(path)) {
+ entry = *entry_it;
+ } else {
+ std::unique_ptr<LogEntry> new_entry(new LogEntry(path));
+ new_entry->newest_parsed_line = line.data() - log.content.data();
+ if (entries_.insert({ new_entry->output, new_entry.get() }).second) {
+ chunk_entries_list.push_back(new_entry.release());
+ return;
+ } else {
+ // Another thread beat us to it. Update the existing entry instead.
+ LogEntry** entry_it = entries_.Lookup(path);
+ assert(entry_it != nullptr);
+ entry = *entry_it;
+ }
}
- }
+ AtomicUpdateMaximum<size_t>(&entry->newest_parsed_line,
+ line.data() - log.content.data());
+ });
+ return chunk_entries_list;
+ });
- // If no newline was found in this chunk, read the next.
- if (!line_end)
- continue;
-
- const char kFieldSeparator = '\t';
-
- char* start = line_start;
- char* end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
-
- int start_time = 0, end_time = 0;
- TimeStamp restat_mtime = 0;
-
- start_time = atoi(start);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
- end_time = atoi(start);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- *end = 0;
- restat_mtime = strtoll(start, NULL, 10);
- start = end + 1;
-
- end = (char*)memchr(start, kFieldSeparator, line_end - start);
- if (!end)
- continue;
- string output = string(start, end - start);
-
- start = end + 1;
- end = line_end;
-
- LogEntry* entry;
- Entries::iterator i = entries_.find(output);
- if (i != entries_.end()) {
- entry = i->second;
- } else {
- entry = new LogEntry(output);
- entries_.insert(Entries::value_type(entry->output, entry));
- ++unique_entry_count;
- }
- ++total_entry_count;
-
- entry->start_time = start_time;
- entry->end_time = end_time;
- entry->mtime = restat_mtime;
- if (log_version >= 5) {
- char c = *end; *end = '\0';
- entry->command_hash = (uint64_t)strtoull(start, NULL, 16);
- *end = c;
- } else {
- entry->command_hash = LogEntry::HashCommand(StringPiece(start,
- end - start));
- }
+ // Collect all the log entries into a flat vector so we can distribute the
+ // final parsing work. This list has a non-deterministic order when a node
+ // appears in multiple chunks.
+ std::vector<LogEntry*> entries_vec;
+ entries_vec.reserve(entries_.size());
+ for (auto& chunk : chunk_new_entries) {
+ std::move(chunk.begin(), chunk.end(), std::back_inserter(entries_vec));
}
- fclose(file);
- if (!line_start) {
- return true; // file was empty
- }
+ // Finish parsing the log entries.
+ ParallelMap(thread_pool.get(), entries_vec, [&log](LogEntry* entry) {
+ // Locate the end of the line. The end of the line wasn't stored in the log
+ // entry because that would complicate the atomic line location updating.
+ assert(entry->newest_parsed_line < log.content.size());
+ StringPiece line_to_end = log.content.substr(entry->newest_parsed_line);
+ StringPiece line;
+ if (!GetNextPiece(&line_to_end, '\n', &line)) {
+ assert(false && "build log changed during parsing");
+ abort();
+ }
+
+ ParsedLine parsed_line {};
+ if (!SplitLine(line, &parsed_line)) {
+ assert(false && "build log changed during parsing");
+ abort();
+ }
+
+ // Initialize the entry object.
+ entry->start_time = atoi(parsed_line.start_time.data());
+ entry->end_time = atoi(parsed_line.end_time.data());
+ entry->mtime = strtoll(parsed_line.mtime.data(), nullptr, 10);
+ entry->command_hash = static_cast<uint64_t>(
+ strtoull(parsed_line.command_hash.data(), nullptr, 16));
+ });
+
+ int total_entry_count = line_count;
+ int unique_entry_count = entries_vec.size();
// Decide whether it's time to rebuild the log:
// - if we're upgrading versions
// - if it's getting large
- int kMinCompactionEntryCount = 100;
- int kCompactionRatio = 3;
- if (log_version < kCurrentVersion) {
+ const int kMinCompactionEntryCount = 100;
+ const int kCompactionRatio = 3;
+ if (log.log_version < kCurrentVersion) {
needs_recompaction_ = true;
} else if (total_entry_count > kMinCompactionEntryCount &&
total_entry_count > unique_entry_count * kCompactionRatio) {
@@ -357,11 +475,10 @@
return true;
}
-BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) {
- Entries::iterator i = entries_.find(path);
- if (i != entries_.end())
- return i->second;
- return NULL;
+BuildLog::LogEntry* BuildLog::LookupByOutput(const HashedStrView& path) {
+ if (LogEntry** i = entries_.Lookup(path))
+ return *i;
+ return nullptr;
}
bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
@@ -388,22 +505,23 @@
return false;
}
- vector<StringPiece> dead_outputs;
- for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) {
- if (user.IsPathDead(i->first)) {
- dead_outputs.push_back(i->first);
- continue;
- }
+ Entries new_entries;
+ new_entries.reserve(std::max(entries_.size(), entries_.bucket_count()));
- if (!WriteEntry(f, *i->second)) {
+ for (const std::pair<HashedStrView, LogEntry*>& pair : entries_) {
+ if (user.IsPathDead(pair.first.str_view()))
+ continue;
+
+ new_entries.insert(pair);
+
+ if (!WriteEntry(f, *pair.second)) {
*err = strerror(errno);
fclose(f);
return false;
}
}
- for (size_t i = 0; i < dead_outputs.size(); ++i)
- entries_.erase(dead_outputs[i]);
+ entries_.swap(new_entries);
fclose(f);
if (unlink(path.c_str()) < 0) {
diff --git a/src/build_log.h b/src/build_log.h
index 5268fab..79bbd74 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -19,7 +19,8 @@
#include <stdio.h>
using namespace std;
-#include "hash_map.h"
+#include "hashed_str_view.h"
+#include "concurrent_hash_map.h"
#include "timestamp.h"
#include "util.h" // uint64_t
@@ -52,12 +53,15 @@
bool Load(const string& path, string* err);
struct LogEntry {
- string output;
+ HashedStr output;
uint64_t command_hash;
int start_time;
int end_time;
TimeStamp mtime;
+ // Used during build log parsing.
+ std::atomic<size_t> newest_parsed_line;
+
static uint64_t HashCommand(StringPiece command);
// Used by tests.
@@ -67,13 +71,13 @@
mtime == o.mtime;
}
- explicit LogEntry(const string& output);
- LogEntry(const string& output, uint64_t command_hash,
+ explicit LogEntry(const HashedStrView& output);
+ LogEntry(const HashedStrView& output, uint64_t command_hash,
int start_time, int end_time, TimeStamp restat_mtime);
};
/// Lookup a previously-run command by its output path.
- LogEntry* LookupByOutput(const string& path);
+ LogEntry* LookupByOutput(const HashedStrView& path);
/// Serialize an entry into a log file.
bool WriteEntry(FILE* f, const LogEntry& entry);
@@ -81,7 +85,10 @@
/// Rewrite the known log entries, throwing away old data.
bool Recompact(const string& path, const BuildLogUser& user, string* err);
- typedef ExternalStringHashMap<LogEntry*>::Type Entries;
+ /// The HashedStrView refers to memory in the LogEntry itself. This map uses
+ /// a value of LogEntry* rather than LogEntry, because moving a LogEntry would
+ /// invalidate the HashedStrView.
+ typedef ConcurrentHashMap<HashedStrView, LogEntry*> Entries;
const Entries& entries() const { return entries_; }
private:
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index ad30380..f5e8c56 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -66,7 +66,7 @@
ASSERT_TRUE(e2);
ASSERT_TRUE(*e1 == *e2);
ASSERT_EQ(15, e1->start_time);
- ASSERT_EQ("out", e1->output);
+ ASSERT_EQ("out", e1->output.str());
}
TEST_F(BuildLogTest, FirstWriteAddsSignature) {
@@ -101,9 +101,9 @@
TEST_F(BuildLogTest, DoubleEntry) {
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "0\t1\t2\tout\tcommand abc\n");
- fprintf(f, "3\t4\t5\tout\tcommand def\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "0\t1\t2\tout\t67ab\n");
+ fprintf(f, "3\t4\t5\tout\t89cd\n");
fclose(f);
string err;
@@ -113,7 +113,7 @@
BuildLog::LogEntry* e = log.LookupByOutput("out");
ASSERT_TRUE(e);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command def", e->command_hash));
+ ASSERT_EQ(0x89cd, e->command_hash);
}
TEST_F(BuildLogTest, Truncate) {
@@ -166,34 +166,19 @@
ASSERT_NE(err.find("version"), string::npos);
}
-TEST_F(BuildLogTest, SpacesInOutputV4) {
- FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout with space\tcommand\n");
- fclose(f);
-
- string err;
- BuildLog log;
- EXPECT_TRUE(log.Load(kTestFilename, &err));
- ASSERT_EQ("", err);
-
- BuildLog::LogEntry* e = log.LookupByOutput("out with space");
- ASSERT_TRUE(e);
- ASSERT_EQ(123, e->start_time);
- ASSERT_EQ(456, e->end_time);
- ASSERT_EQ(456, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
-}
-
TEST_F(BuildLogTest, DuplicateVersionHeader) {
// Old versions of ninja accidentally wrote multiple version headers to the
// build log on Windows. This shouldn't crash, and the second version header
// should be ignored.
+ //
+ // This test initially tested a v4 log, but support for v4 was removed. It now
+ // tests a v5 log, but that doesn't mean that the original duplicate-header
+ // bug ever affected v5 logs.
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout\tcommand\n");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "456\t789\t789\tout2\tcommand2\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "123\t456\t456\tout\t12ab\n");
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "456\t789\t789\tout2\t34cd\n");
fclose(f);
string err;
@@ -206,26 +191,27 @@
ASSERT_EQ(123, e->start_time);
ASSERT_EQ(456, e->end_time);
ASSERT_EQ(456, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command", e->command_hash));
+ ASSERT_EQ(0x12ab, e->command_hash);
e = log.LookupByOutput("out2");
ASSERT_TRUE(e);
ASSERT_EQ(456, e->start_time);
ASSERT_EQ(789, e->end_time);
ASSERT_EQ(789, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
+ ASSERT_EQ(0x34cd, e->command_hash);
}
TEST_F(BuildLogTest, VeryLongInputLine) {
- // Ninja's build log buffer is currently 256kB. Lines longer than that are
- // silently ignored, but don't affect parsing of other lines.
+ // The parallelized build log parser accepts paths of arbitrary length.
+ std::string very_long_path;
+ for (size_t i = 0; i < (512 << 10) / strlen("/more_path"); ++i)
+ very_long_path += "/more_path";
+
FILE* f = fopen(kTestFilename, "wb");
- fprintf(f, "# ninja log v4\n");
- fprintf(f, "123\t456\t456\tout\tcommand start");
- for (size_t i = 0; i < (512 << 10) / strlen(" more_command"); ++i)
- fputs(" more_command", f);
+ fprintf(f, "# ninja log v5\n");
+ fprintf(f, "123\t456\t456\t%s\t1234abcd\n", very_long_path.c_str());
fprintf(f, "\n");
- fprintf(f, "456\t789\t789\tout2\tcommand2\n");
+ fprintf(f, "456\t789\t789\tout2\t2345bcde\n");
fclose(f);
string err;
@@ -233,15 +219,19 @@
EXPECT_TRUE(log.Load(kTestFilename, &err));
ASSERT_EQ("", err);
- BuildLog::LogEntry* e = log.LookupByOutput("out");
- ASSERT_EQ(NULL, e);
+ BuildLog::LogEntry* e = log.LookupByOutput(very_long_path.c_str());
+ ASSERT_TRUE(e);
+ ASSERT_EQ(123, e->start_time);
+ ASSERT_EQ(456, e->end_time);
+ ASSERT_EQ(456, e->mtime);
+ ASSERT_EQ(0x1234abcd, e->command_hash);
e = log.LookupByOutput("out2");
ASSERT_TRUE(e);
ASSERT_EQ(456, e->start_time);
ASSERT_EQ(789, e->end_time);
ASSERT_EQ(789, e->mtime);
- ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
+ ASSERT_EQ(0x2345bcde, e->command_hash);
}
TEST_F(BuildLogTest, MultiTargetEdge) {
@@ -256,8 +246,8 @@
ASSERT_TRUE(e1);
BuildLog::LogEntry* e2 = log.LookupByOutput("out.d");
ASSERT_TRUE(e2);
- ASSERT_EQ("out", e1->output);
- ASSERT_EQ("out.d", e2->output);
+ ASSERT_EQ("out", e1->output.str());
+ ASSERT_EQ("out.d", e2->output.str());
ASSERT_EQ(21, e1->start_time);
ASSERT_EQ(21, e2->start_time);
ASSERT_EQ(22, e2->end_time);
@@ -297,11 +287,11 @@
// "out2" is dead, it should've been removed.
BuildLog log3;
- EXPECT_TRUE(log2.Load(kTestFilename, &err));
+ EXPECT_TRUE(log3.Load(kTestFilename, &err));
ASSERT_EQ("", err);
- ASSERT_EQ(1u, log2.entries().size());
- ASSERT_TRUE(log2.LookupByOutput("out"));
- ASSERT_FALSE(log2.LookupByOutput("out2"));
+ ASSERT_EQ(1u, log3.entries().size());
+ ASSERT_TRUE(log3.LookupByOutput("out"));
+ ASSERT_FALSE(log3.LookupByOutput("out2"));
}
} // anonymous namespace
diff --git a/src/build_test.cc b/src/build_test.cc
index 46ab33e..f7c8550 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -19,6 +19,7 @@
#include "build_log.h"
#include "deps_log.h"
#include "graph.h"
+#include "status.h"
#include "test.h"
/// Fixture for tests involving Plan.
@@ -455,10 +456,55 @@
VirtualFileSystem* fs_;
};
+/// Fake implementation of Status, useful for tests.
+struct FakeStatus : public Status {
+ explicit FakeStatus() : last_output_(), warnings_() {}
+
+ virtual void PlanHasTotalEdges(int total) {}
+ virtual void BuildEdgeStarted(Edge* edge, int64_t start_time_millis) {}
+ virtual void BuildStarted() {}
+ virtual void BuildFinished() {}
+
+ virtual void Debug(const char* msg, ...) {}
+ virtual void Info(const char* msg, ...) {}
+ virtual void Error(const char* msg, ...) {}
+
+ virtual void BuildEdgeFinished(Edge* edge, int64_t end_time_millis,
+ const CommandRunner::Result* result) {
+ last_output_ = result->output;
+ }
+
+ virtual void Warning(const char* msg, ...) {
+ va_list ap, ap2;
+ va_start(ap, msg);
+ va_copy(ap2, ap);
+
+ int len = vsnprintf(NULL, 0, msg, ap2);
+ va_end(ap2);
+ if (len < 0) {
+ warnings_.push_back("vsnprintf failed");
+ return;
+ }
+
+ string buf;
+ buf.resize(len + 1);
+
+ len = vsnprintf(&buf[0], len + 1, msg, ap);
+ buf.resize(len);
+
+ warnings_.push_back(buf);
+
+ va_end(ap);
+ }
+
+ string last_output_;
+ vector<string> warnings_;
+};
+
struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser {
BuildTest() : config_(MakeConfig()), command_runner_(&fs_),
- builder_(&state_, config_, NULL, NULL, &fs_),
- status_(config_) {
+ builder_(&state_, config_, NULL, NULL, &fs_, &status_, 0),
+ status_() {
}
virtual void SetUp() {
@@ -501,7 +547,7 @@
VirtualFileSystem fs_;
Builder builder_;
- BuildStatus status_;
+ FakeStatus status_;
};
void BuildTest::RebuildTarget(const string& target, const char* manifest,
@@ -525,12 +571,12 @@
DepsLog deps_log, *pdeps_log = NULL;
if (deps_path) {
ASSERT_TRUE(deps_log.Load(deps_path, pstate, &err));
- ASSERT_TRUE(deps_log.OpenForWrite(deps_path, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite(deps_path, fs_, &err));
ASSERT_EQ("", err);
pdeps_log = &deps_log;
}
- Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_);
+ Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, &status_, 0);
EXPECT_TRUE(builder.AddTarget(target, &err));
command_runner_.commands_ran_.clear();
@@ -564,7 +610,9 @@
} else if (edge->rule().name() == "true" ||
edge->rule().name() == "fail" ||
edge->rule().name() == "interrupt" ||
- edge->rule().name() == "console") {
+ edge->rule().name() == "console" ||
+ edge->rule().name() == "mkdir" ||
+ edge->rule().name() == "phony_out") {
// Don't do anything.
} else {
printf("unknown command\n");
@@ -597,6 +645,18 @@
return true;
}
+ if (edge->rule().name() == "mkdir") {
+ result->status = ExitSuccess;
+ for (vector<Node*>::iterator out = edge->outputs_.begin();
+ out != edge->outputs_.end(); ++out) {
+ if (!fs_->MakeDir((*out)->path())) {
+ result->status = ExitFailure;
+ }
+ }
+ last_command_ = NULL;
+ return true;
+ }
+
if (edge->rule().name() == "fail" ||
(edge->rule().name() == "touch-fail-tick2" && fs_->now_ == 2))
result->status = ExitFailure;
@@ -855,6 +915,23 @@
ASSERT_EQ("cc foo.c", edge->EvaluateCommand());
}
+TEST_F(BuildTest, DepFileOKWithPhonyOutputs) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule cc\n command = cc $in\n depfile = $out.d\n"
+"build foo.o: cc foo.c\n"));
+
+ config_.uses_phony_outputs = true;
+
+ fs_.Create("foo.c", "");
+ GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing.
+ fs_.Create("foo.o.d", "foo.o: blah.h bar.h\n");
+ EXPECT_TRUE(builder_.AddTarget("foo.o", &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_TRUE(GetNode("bar.h")->in_edge()->phony_from_depfile_);
+}
+
TEST_F(BuildTest, DepFileParseError) {
string err;
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
@@ -875,7 +952,7 @@
"build b: touch || c\n"
"build a: touch | b || c\n"));
- vector<Edge*> c_out = GetNode("c")->out_edges();
+ vector<Edge*> c_out = GetNode("c")->GetOutEdges();
ASSERT_EQ(2u, c_out.size());
EXPECT_EQ("b", c_out[0]->outputs_[0]->path());
EXPECT_EQ("a", c_out[1]->outputs_[0]->path());
@@ -1324,8 +1401,8 @@
ASSERT_EQ("", err);
EXPECT_TRUE(builder_.Build(&err));
ASSERT_EQ("", err);
- EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]",
- BuildStatus::kEdgeStarted));
+ EXPECT_EQ(3u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(3u, builder_.plan_.command_edge_count());
command_runner_.commands_ran_.clear();
state_.Reset();
@@ -1763,20 +1840,6 @@
ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
-TEST_F(BuildTest, StatusFormatElapsed) {
- status_.BuildStarted();
- // Before any task is done, the elapsed time must be zero.
- EXPECT_EQ("[%/e0.000]",
- status_.FormatProgressStatus("[%%/e%e]",
- BuildStatus::kEdgeStarted));
-}
-
-TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
- EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
- status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]",
- BuildStatus::kEdgeStarted));
-}
-
TEST_F(BuildTest, FailedDepsParse) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build bad_deps.o: cat in1\n"
@@ -1833,10 +1896,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -1864,9 +1927,9 @@
// Run the build again.
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -1904,10 +1967,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -1934,9 +1997,9 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -1972,7 +2035,7 @@
// The deps log is NULL in dry runs.
config_.dry_run = true;
- Builder builder(&state, config_, NULL, NULL, &fs_);
+ Builder builder(&state, config_, NULL, NULL, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
@@ -2027,10 +2090,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("out", &err));
ASSERT_EQ("", err);
@@ -2054,9 +2117,9 @@
// Run the build again.
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
command_runner_.commands_ran_.clear();
EXPECT_TRUE(builder.AddTarget("out", &err));
@@ -2086,10 +2149,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("fo o.o", &err));
ASSERT_EQ("", err);
@@ -2107,10 +2170,10 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
Edge* edge = state.edges_.back();
@@ -2148,10 +2211,10 @@
// Run the build once, everything should be ok.
DepsLog deps_log;
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err));
ASSERT_EQ("", err);
@@ -2171,10 +2234,10 @@
DepsLog deps_log;
ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
- ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
ASSERT_EQ("", err);
- Builder builder(&state, config_, NULL, &deps_log, &fs_);
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner_);
Edge* edge = state.edges_.back();
@@ -2308,3 +2371,518 @@
EXPECT_EQ("", err);
ASSERT_EQ(1u, command_runner_.commands_ran_.size());
}
+
+TEST_F(BuildTest, OutputDirectoryIgnored) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputDirectoryWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ config_.uses_phony_outputs = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("ninja: outputs should be files, not directories: outdir", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputDirectoryError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule mkdir\n"
+" command = mkdir $out\n"
+"build outdir: mkdir\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.output_directory_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outdir", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: outputs should be files, not directories: outdir", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingIgnore) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ config_.uses_phony_outputs = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("ninja: output file missing after successful execution: outfile", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileMissingError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build outfile: true\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.missing_output_file_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outfile", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: output file missing after successful execution: outfile", status_.last_output_);
+}
+
+TEST_F(BuildTest, OutputFileNotNeeded) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build outphony: phony_out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.missing_output_file_should_err = true;
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("outphony", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileIgnored) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ config_.uses_phony_outputs = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("ninja: Missing `restat`? An output file is older than the most recent input:\n output: out\n input: in", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OldOutputFileError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule true\n command = true\n"
+"build out: true in\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.old_output_should_err = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_FALSE(builder_.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("ninja: Missing `restat`? An output file is older than the most recent input:\n output: out\n input: in", status_.last_output_);
+}
+
+TEST_F(BuildWithLogTest, OutputFileUpdated) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n command = touch ${out}\n"
+"build out: touch in\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.old_output_should_err = true;
+
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ("", status_.last_output_);
+}
+
+TEST_F(BuildWithDepsLogTest, MissingDepfileWarning) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out1: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ string err;
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state_, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+
+ ASSERT_EQ(1u, status_.warnings_.size());
+ ASSERT_EQ("depfile is missing (in1.d for out1)", status_.warnings_[0]);
+
+ builder.command_runner_.release();
+}
+
+TEST_F(BuildWithDepsLogTest, MissingDepfileError) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out1: cat in1\n"
+" deps = gcc\n"
+" depfile = in1.d\n"));
+
+ config_.missing_depfile_should_err = true;
+
+ string err;
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state_, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out1", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(builder.Build(&err));
+ EXPECT_EQ("subcommand failed", err);
+
+ EXPECT_EQ("depfile is missing", status_.last_output_);
+
+ builder.command_runner_.release();
+}
+
+TEST_F(BuildTest, PreRemoveOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule touch\n command = touch ${out}\n"
+"build out: touch in\n"
+"build out2: touch out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.pre_remove_output_files = true;
+
+ fs_.Create("out", "");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out2", &err));
+ EXPECT_EQ("", err);
+
+ fs_.files_created_.clear();
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(2u, fs_.files_created_.size());
+ EXPECT_EQ(1u, fs_.files_removed_.size());
+}
+
+TEST_F(BuildTest, PreRemoveOutputsWithPhonyOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build out: phony_out\n"));
+
+ config_.uses_phony_outputs = true;
+ config_.pre_remove_output_files = true;
+
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ fs_.files_created_.clear();
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+ EXPECT_EQ(0u, fs_.files_created_.size());
+ EXPECT_EQ(0u, fs_.files_removed_.size());
+}
+
+TEST_F(BuildTest, Validation) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2\n"));
+
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in" only rebuilds "out" ("validate" doesn't depend on
+ // "out").
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+
+ // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on
+ // "validate").
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildTest, ValidationDependsOnOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2 | out\n"));
+
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+
+ string err;
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ EXPECT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in" rebuilds "out" and "validate".
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(2u, command_runner_.commands_ran_.size());
+
+ // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on
+ // "validate").
+ fs_.Tick();
+ fs_.Create("in2", "");
+
+ err.clear();
+ command_runner_.commands_ran_.clear();
+ state_.Reset();
+ EXPECT_TRUE(builder_.AddTarget("out", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder_.Build(&err));
+ EXPECT_EQ("", err);
+
+ EXPECT_EQ(1u, command_runner_.commands_ran_.size());
+}
+
+TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) {
+ const char* manifest =
+ "build out: cat in |@ validate\n"
+ "build validate: cat in2 | out\n"
+ "build out2: cat in3\n"
+ " deps = gcc\n"
+ " depfile = out2.d\n";
+
+ string err;
+
+ {
+ fs_.Create("in", "");
+ fs_.Create("in2", "");
+ fs_.Create("in3", "");
+ fs_.Create("out2.d", "out: out");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // On the first build, only the out2 command is run.
+ EXPECT_EQ(command_runner_.commands_ran_.size(), 1);
+
+ // The deps file should have been removed.
+ EXPECT_EQ(0, fs_.Stat("out2.d", &err));
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+
+ fs_.Tick();
+ command_runner_.commands_ran_.clear();
+
+ {
+ fs_.Create("in2", "");
+ fs_.Create("in3", "");
+
+ State state;
+ ASSERT_NO_FATAL_FAILURE(AddCatRule(&state));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest));
+
+ DepsLog deps_log;
+ ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err));
+ ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", fs_, &err));
+ ASSERT_EQ("", err);
+
+ Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0);
+ builder.command_runner_.reset(&command_runner_);
+
+ EXPECT_TRUE(builder.AddTarget("out2", &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(builder.Build(&err));
+ EXPECT_EQ("", err);
+
+ // The out and validate actions should have been run as well as out2.
+ ASSERT_EQ(command_runner_.commands_ran_.size(), 3);
+ EXPECT_EQ(command_runner_.commands_ran_[0], "cat in > out");
+ EXPECT_EQ(command_runner_.commands_ran_[1], "cat in2 > validate");
+ EXPECT_EQ(command_runner_.commands_ran_[2], "cat in3 > out2");
+
+ deps_log.Close();
+ builder.command_runner_.release();
+ }
+}
diff --git a/src/clean.cc b/src/clean.cc
index ce6a575..d3c2d56 100644
--- a/src/clean.cc
+++ b/src/clean.cc
@@ -116,10 +116,10 @@
for (vector<Edge*>::iterator e = state_->edges_.begin();
e != state_->edges_.end(); ++e) {
// Do not try to remove phony targets
- if ((*e)->is_phony())
+ if ((*e)->is_phony() || (*e)->IsPhonyOutput())
continue;
// Do not remove generator's files unless generator specified.
- if (!generator && (*e)->GetBindingBool("generator"))
+ if (!generator && (*e)->IsGenerator())
continue;
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
@@ -135,7 +135,7 @@
void Cleaner::DoCleanTarget(Node* target) {
if (Edge* e = target->in_edge()) {
// Do not try to remove phony targets
- if (!e->is_phony()) {
+ if (!e->is_phony() && !e->IsPhonyOutput()) {
Remove(target->path());
RemoveEdgeFiles(e);
}
@@ -209,6 +209,9 @@
for (vector<Edge*>::iterator e = state_->edges_.begin();
e != state_->edges_.end(); ++e) {
if ((*e)->rule().name() == rule->name()) {
+ if ((*e)->IsPhonyOutput()) {
+ continue;
+ }
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
@@ -232,7 +235,8 @@
assert(rule);
Reset();
- const Rule* r = state_->bindings_.LookupRule(rule);
+ ScopePosition end_of_root { &state_->root_scope_, kLastDeclIndex };
+ const Rule* r = Scope::LookupRuleAtPos(rule, end_of_root);
if (r) {
CleanRule(r);
} else {
@@ -249,7 +253,8 @@
PrintHeader();
for (int i = 0; i < rule_count; ++i) {
const char* rule_name = rules[i];
- const Rule* rule = state_->bindings_.LookupRule(rule_name);
+ ScopePosition end_of_root { &state_->root_scope_, kLastDeclIndex };
+ const Rule* rule = Scope::LookupRuleAtPos(rule_name, end_of_root);
if (rule) {
if (IsVerbose())
printf("Rule %s\n", rule_name);
diff --git a/src/clean_test.cc b/src/clean_test.cc
index 395343b..4fdaa96 100644
--- a/src/clean_test.cc
+++ b/src/clean_test.cc
@@ -377,6 +377,35 @@
EXPECT_LT(0, fs_.Stat("phony", &err));
}
+TEST_F(CleanTest, CleanPhonyOutput) {
+ string err;
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build phout: phony_out t1 t2\n"
+"build t1: cat\n"
+"build t2: cat\n"));
+
+ fs_.Create("phout", "");
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
+
+ // Check that CleanAll does not remove "phout".
+ Cleaner cleaner(&state_, config_, &fs_);
+ EXPECT_EQ(0, cleaner.CleanAll());
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ EXPECT_LT(0, fs_.Stat("phout", &err));
+
+ fs_.Create("t1", "");
+ fs_.Create("t2", "");
+
+ // Check that CleanTarget does not remove "phony".
+ EXPECT_EQ(0, cleaner.CleanTarget("phout"));
+ EXPECT_EQ(2, cleaner.cleaned_files_count());
+ EXPECT_LT(0, fs_.Stat("phout", &err));
+}
+
TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"rule cc_dep\n"
diff --git a/src/concurrent_hash_map.h b/src/concurrent_hash_map.h
new file mode 100644
index 0000000..6e5d660
--- /dev/null
+++ b/src/concurrent_hash_map.h
@@ -0,0 +1,274 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_CONCURRENT_HASH_MAP_
+#define NINJA_CONCURRENT_HASH_MAP_
+
+#include <stdint.h>
+
+#include <atomic>
+#include <memory>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "metrics.h"
+#include "util.h"
+
+/// A hash table that allows for concurrent lookups and insertions.
+///
+/// Resizing the table isn't thread-safe; inserting items doesn't automatically
+/// resize the table.
+template <typename K, typename V>
+class ConcurrentHashMap {
+public:
+ using key_type = K;
+ using mapped_type = V;
+ using value_type = std::pair<const K, V>;
+ using reference = value_type&;
+ using pointer = value_type*;
+ using const_reference = const value_type&;
+ using const_pointer = const value_type*;
+
+private:
+ struct Node;
+
+ struct NodeStub {
+ std::atomic<Node*> next { nullptr };
+ };
+
+ struct Node : NodeStub {
+ Node(size_t hash, const value_type& value) : hash(hash), value(value) {}
+
+ size_t hash = 0;
+ value_type value {};
+ };
+
+ /// Record the number of elements in the hash map. Assign each thread to a
+ /// separate index within the size table to avoid false sharing among CPUs.
+ struct NINJA_ALIGNAS_CACHE_LINE SizeRecord {
+ std::atomic<ssize_t> size;
+ };
+ std::vector<SizeRecord> size_ { GetThreadSlotCount() };
+
+ void IncrementSize() {
+ thread_local size_t idx = GetThreadSlotIndex();
+ size_[idx].size++;
+ }
+
+public:
+ explicit ConcurrentHashMap(size_t size=1)
+ : table_(std::max<size_t>(size, 1)) {}
+
+ ConcurrentHashMap(const ConcurrentHashMap&) = delete;
+ ConcurrentHashMap& operator=(const ConcurrentHashMap&) = delete;
+
+ ~ConcurrentHashMap() {
+ ForEachNode(table_, [](Node* node) {
+ delete node;
+ });
+ }
+
+ void swap(ConcurrentHashMap& other) {
+ size_.swap(other.size_);
+ table_.swap(other.table_);
+ }
+
+ /// Atomically lookup a key.
+ V* Lookup(const K& key) {
+ std::pair<NodeStub*, Node*> pos = Find(std::hash<K>()(key), key);
+ return pos.first ? nullptr : &pos.second->value.second;
+ }
+
+ /// Atomically lookup a key.
+ const V* Lookup(const K& key) const {
+ auto self = const_cast<ConcurrentHashMap*>(this);
+ std::pair<NodeStub*, Node*> pos = self->Find(std::hash<K>()(key), key);
+ return pos.first ? nullptr : &pos.second->value.second;
+ }
+
+ /// Atomically insert a (K, V) entry into the hash map if K isn't already in
+ /// the map. Does nothing if K is already in the map.
+ ///
+ /// Returns the address of the V entry in the map, and a bool denoting whether
+ /// or not an insertion was performed.
+ ///
+ /// This function avoids returning an iterator because insertion is
+ /// thread-safe while iteration isn't, so returning an iterator seemed
+ /// hazardous.
+ std::pair<V*, bool> insert(const value_type& value) {
+ size_t hash = std::hash<K>()(value.first);
+ std::unique_ptr<Node> node(new Node(hash, value));
+ std::pair<Node*, bool> result = InsertNode(std::move(node));
+ return { &result.first->value.second, result.second };
+ }
+
+ size_t size() const {
+ ssize_t result = 0;
+ for (const SizeRecord& rec : size_)
+ result += rec.size.load();
+ return result;
+ }
+ bool empty() const { return size() == 0; }
+ size_t bucket_count() const { return table_.size(); }
+
+ /// Unlike insertion and lookup, rehashing is *not* thread-safe. This class
+ /// does not automatically rehash the table on insertion.
+ void rehash(size_t buckets) {
+ METRIC_RECORD("ConcurrentHashMap rehash");
+
+ std::vector<NodeStub> tmp_table(std::max<size_t>(buckets, 1));
+ tmp_table.swap(table_);
+ ForEachNode(tmp_table, [this](Node* node) {
+ node->next = nullptr;
+ InsertNode(std::unique_ptr<Node>(node));
+ });
+ }
+
+ void reserve(size_t count) {
+ if (count < bucket_count())
+ return;
+ rehash(count);
+ }
+
+private:
+ template <typename Func>
+ void ForEachNode(const std::vector<NodeStub>& table, Func&& func) {
+ for (const NodeStub& stub : table) {
+ Node* node = stub.next.load();
+ while (node != nullptr) {
+ Node* next = node->next.load();
+ func(node);
+ node = next;
+ }
+ }
+ }
+
+ /// Insert a node into the hash map if it isn't already there.
+ std::pair<Node*, bool> InsertNode(std::unique_ptr<Node> node) {
+ while (true) {
+ std::pair<NodeStub*, Node*> pos = Find(node->hash, node->value.first);
+ if (pos.first == nullptr)
+ return { pos.second, false };
+
+ node->next = pos.second;
+ Node* expected = pos.second;
+ if (pos.first->next.compare_exchange_weak(expected, node.get())) {
+ IncrementSize();
+ return { node.release(), true };
+ }
+ }
+ }
+
+ std::vector<NodeStub> table_;
+
+ /// Find either a matching node or the insertion point for the node. To ensure
+ /// deterministic behavior, the nodes are kept in ascending order, first by
+ /// the hash value, then by the key itself.
+ ///
+ /// This function returns a pair of (stub, node):
+ /// - If the node is found, stub is null and node is non-null.
+ /// - If the node isn't found, stub is non-null, and node is the loaded value
+ /// of the stub, which should be used in the CAS to insert the node.
+ std::pair<NodeStub*, Node*> Find(size_t hash, const K& key) {
+ NodeStub* stub = &table_[hash % table_.size()];
+ while (true) {
+ Node* ptr = stub->next.load();
+ if (ptr == nullptr || hash < ptr->hash) {
+ return { stub, ptr };
+ }
+ if (hash == ptr->hash) {
+ // In the common case, when the hash and ptr->hash are equal, we expect
+ // the keys to be equal too, so compare for equality before comparing
+ // for less-than.
+ if (key == ptr->value.first) {
+ return { nullptr, ptr };
+ } else if (key < ptr->value.first) {
+ return { stub, ptr };
+ }
+ }
+ stub = ptr;
+ }
+ }
+
+ struct MutCfg { using ValueType = value_type; };
+ struct ConstCfg { using ValueType = const value_type; };
+
+ /// Something like a ForwardIterator. Iteration over the table isn't
+ /// thread-safe.
+ template <typename Cfg>
+ class iterator_impl {
+ iterator_impl(const ConcurrentHashMap* container) : container_(container) {}
+ iterator_impl(const ConcurrentHashMap* container, size_t bucket, Node* node)
+ : container_(container),
+ bucket_(bucket),
+ node_(node) {}
+
+ const ConcurrentHashMap* container_ = nullptr;
+ size_t bucket_ = SIZE_MAX;
+ Node* node_ = nullptr;
+
+ public:
+ /// This constructor is a copy constructor for const_iterator and an
+ /// implicit conversion for const_iterator -> iterator.
+ iterator_impl(const iterator_impl<ConstCfg>& other)
+ : container_(other.container_),
+ bucket_(other.bucket_),
+ node_(other.node_) {}
+
+ typename Cfg::ValueType& operator*() { return node_->value; }
+ typename Cfg::ValueType* operator->() { return &node_->value; }
+
+ iterator_impl<Cfg>& operator++() {
+ node_ = node_->next.load();
+ if (node_ == nullptr)
+ *this = container_->FindNextOccupiedBucket(bucket_ + 1);
+ return *this;
+ }
+
+ bool operator==(const iterator_impl<Cfg>& other) const {
+ return node_ == other.node_;
+ }
+ bool operator!=(const iterator_impl<Cfg>& other) const {
+ return node_ != other.node_;
+ }
+
+ friend class ConcurrentHashMap;
+ };
+
+public:
+ using iterator = iterator_impl<MutCfg>;
+ using const_iterator = iterator_impl<ConstCfg>;
+
+ iterator begin() { return FindNextOccupiedBucket(0); }
+ iterator end() { return iterator { this }; }
+ const_iterator begin() const { return FindNextOccupiedBucket(0); }
+ const_iterator end() const { return const_iterator { this }; }
+ const_iterator cbegin() const { return FindNextOccupiedBucket(0); }
+ const_iterator cend() const { return const_iterator { this }; }
+
+private:
+ const_iterator FindNextOccupiedBucket(size_t idx) const {
+ while (idx < table_.size()) {
+ Node* node = table_[idx].next.load();
+ if (node != nullptr) {
+ return const_iterator { this, idx, node };
+ }
+ ++idx;
+ }
+ return end();
+ }
+};
+
+#endif // NINJA_CONCURRENT_HASH_MAP_
diff --git a/src/debug_flags.cc b/src/debug_flags.cc
index 44b14c4..e77d351 100644
--- a/src/debug_flags.cc
+++ b/src/debug_flags.cc
@@ -19,3 +19,5 @@
bool g_keep_rsp = false;
bool g_experimental_statcache = true;
+
+bool g_use_threads = true;
diff --git a/src/debug_flags.h b/src/debug_flags.h
index e08a43b..d2d57ee 100644
--- a/src/debug_flags.h
+++ b/src/debug_flags.h
@@ -30,4 +30,6 @@
extern bool g_experimental_statcache;
+extern bool g_use_threads;
+
#endif // NINJA_EXPLAIN_H_
diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc
index 405289f..16403f1 100644
--- a/src/depfile_parser.cc
+++ b/src/depfile_parser.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 1.1.1 */
+/* Generated by re2c 1.3 */
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/src/deps_log.cc b/src/deps_log.cc
index 0bb96f3..807dbd3 100644
--- a/src/deps_log.cc
+++ b/src/deps_log.cc
@@ -25,14 +25,21 @@
typedef unsigned __int32 uint32_t;
#endif
+#include <numeric>
+
+#include "disk_interface.h"
#include "graph.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
// The version is stored as 4 bytes after the signature and also serves as a
// byte order mark. Signature and version combined are 16 bytes long.
-const char kFileSignature[] = "# ninjadeps\n";
+static constexpr StringPiece kFileSignature { "# ninjadeps\n", 12 };
+static_assert(kFileSignature.size() % 4 == 0,
+ "file signature size is not a multiple of 4");
+static constexpr size_t kFileHeaderSize = kFileSignature.size() + 4;
const int kCurrentVersion = 4;
// Record size is currently limited to less than the full 32 bit, due to
@@ -43,9 +50,9 @@
Close();
}
-bool DepsLog::OpenForWrite(const string& path, string* err) {
+bool DepsLog::OpenForWrite(const string& path, const DiskInterface& disk, string* err) {
if (needs_recompaction_) {
- if (!Recompact(path, err))
+ if (!Recompact(path, disk, err))
return false;
}
@@ -64,7 +71,7 @@
fseek(file_, 0, SEEK_END);
if (ftell(file_) == 0) {
- if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) {
+ if (fwrite(kFileSignature.data(), kFileSignature.size(), 1, file_) < 1) {
*err = strerror(errno);
return false;
}
@@ -167,123 +174,481 @@
file_ = NULL;
}
-bool DepsLog::Load(const string& path, State* state, string* err) {
- METRIC_RECORD(".ninja_deps load");
- char buf[kMaxRecordSize + 1];
- FILE* f = fopen(path.c_str(), "rb");
- if (!f) {
- if (errno == ENOENT)
- return true;
- *err = strerror(errno);
+struct DepsLogWordSpan {
+ size_t begin; // starting index into a deps log buffer of uint32 words
+ size_t end; // stopping index
+};
+
+/// Split the v4 deps log into independently-parseable chunks using a heuristic.
+/// If the heuristic fails, we'll still load the file correctly, but it could be
+/// slower.
+///
+/// There are two kinds of records -- path and deps records. Their formats:
+///
+/// path:
+/// - uint32 size -- high bit is clear
+/// - String content. The string is padded to a multiple of 4 bytes with
+/// trailing NULs.
+/// - uint32 checksum (ones complement of the path's index / node ID)
+///
+/// deps:
+/// - uint32 size -- high bit is set
+/// - int32 output_path_id
+/// - uint32 output_path_mtime_lo
+/// - uint32 output_path_mtime_hi
+/// - int32 input_path_id[...] -- every remaining word is an input ID
+///
+/// To split the deps log into chunks, look for uint32 words with the value
+/// 0x8000xxxx, where xxxx is nonzero. Such a word is almost guaranteed to be
+/// the size field of a deps record (with fewer than ~16K dependencies):
+/// - It can't be part of a string, because paths can't have embedded NULs.
+/// - It (probably) can't be a node ID, because node IDs are represented using
+/// "int", and it would be unlikely to have more than 2 billion of them. An
+/// Android build typically has about 1 million nodes.
+/// - It's unlikely to be part of a path checksum, because that would also
+/// imply that we have at least 2 billion nodes.
+/// - It could be the upper word of a mtime from 2262, or the lower word, which
+/// wraps every ~4s. We rule these out by looking for the mtime's deps size
+/// two or three words above the split candidate.
+///
+/// This heuristic can fail in a few ways:
+/// - We only find path records in the area we scan.
+/// - The deps records all have >16K of dependencies. (Almost all deps records
+/// I've seen in the Android build have a few hundred. Only a few have ~10K.)
+/// - All deps records with <16K of dependencies are preceded by something that
+/// the mtime check rejects:
+/// - A deps record with an mtime within a 524us span every 4s, with zero or
+/// one dependency.
+/// - A deps record with an mtime from 2262, with one or two dependencies.
+/// - A path record containing "xx xx 0[0-7] 80" (little-endian) or
+/// "80 0[0-7] xx xx" (big-endian) as the last or second-to-last word. The
+/// "0[0-7]" is a control character, and "0[0-7] 80" is invalid UTF-8.
+///
+/// Maybe we can add a delimiter to the log format and replace this heuristic.
+size_t MustBeDepsRecordHeader(DepsLogData log, size_t index) {
+ assert(index < log.size);
+
+ size_t this_header = log.words[index];
+ if ((this_header & 0xffff0000) != 0x80000000) return false;
+ if ((this_header & 0x0000ffff) == 0) return false;
+
+ // We've either found a deps record or an mtime. If it's an mtime, the
+ // word two or three spaces back will be a valid deps size (0x800xxxxx).
+ auto might_be_deps_record_header = [](size_t header) {
+ return (header & 0x80000000) == 0x80000000 &&
+ (header & 0x7fffffff) <= kMaxRecordSize;
+ };
+ if (index >= 3 && might_be_deps_record_header(log.words[index - 3])) {
+ return false;
+ }
+ if (index >= 2 && might_be_deps_record_header(log.words[index - 2])) {
return false;
}
- bool valid_header = true;
+ // Success: As long as the deps log is valid up to this point, this index
+ // must start a deps record.
+ return true;
+}
+
+static std::vector<DepsLogWordSpan>
+SplitDepsLog(DepsLogData log, ThreadPool* thread_pool) {
+ if (log.size == 0) return {};
+
+ std::vector<std::pair<size_t, size_t>> blind_splits =
+ SplitByThreads(log.size);
+ std::vector<DepsLogWordSpan> chunks;
+ size_t chunk_start = 0;
+
+ auto split_candidates = ParallelMap(thread_pool, blind_splits,
+ [log](std::pair<size_t, size_t> chunk) {
+ for (size_t i = chunk.first; i < chunk.second; ++i) {
+ if (MustBeDepsRecordHeader(log, i)) {
+ return i;
+ }
+ }
+ return SIZE_MAX;
+ });
+ for (size_t candidate : split_candidates) {
+ assert(chunk_start <= candidate);
+ if (candidate != SIZE_MAX && chunk_start < candidate) {
+ chunks.push_back({ chunk_start, candidate });
+ chunk_start = candidate;
+ }
+ }
+
+ assert(chunk_start < log.size);
+ chunks.push_back({ chunk_start, log.size });
+ return chunks;
+}
+
+struct InputRecord {
+ enum Kind { InvalidHeader, PathRecord, DepsRecord } kind = InvalidHeader;
+ size_t size = 0; // number of 32-bit words, including the header
+
+ union {
+ struct {
+ StringPiece path;
+ int checksum;
+ } path;
+
+ struct {
+ int output_id;
+ TimeStamp mtime;
+ const uint32_t* deps;
+ size_t deps_count;
+ } deps;
+ } u = {};
+};
+
+/// Parse a deps log record at the given index within the loaded deps log.
+/// If there is anything wrong with the header (e.g. the record extends past the
+/// end of the file), this function returns a blank InvalidHeader record.
+static InputRecord ParseRecord(DepsLogData log, size_t index) {
+ InputRecord result {};
+ assert(index < log.size);
+
+ // The header of a record is the size of the record, in bytes, without
+ // including the header itself. The high bit is set for a deps record and
+ // clear for a path record.
+ const uint32_t header = log.words[index];
+ const uint32_t raw_size = header & 0x7fffffff;
+ if (raw_size % sizeof(uint32_t) != 0) return result;
+ if (raw_size > kMaxRecordSize) return result;
+
+ // Add the header to the size for easier iteration over records later on.
+ const size_t size = raw_size / sizeof(uint32_t) + 1;
+ if (log.size - index < size) return result;
+
+ if ((header & 0x80000000) == 0) {
+ // Path record (header, content, checksum).
+ if (size < 3) return result;
+
+ const char* path = reinterpret_cast<const char*>(&log.words[index + 1]);
+ size_t path_size = (size - 2) * sizeof(uint32_t);
+ if (path[path_size - 1] == '\0') --path_size;
+ if (path[path_size - 1] == '\0') --path_size;
+ if (path[path_size - 1] == '\0') --path_size;
+
+ result.kind = InputRecord::PathRecord;
+ result.size = size;
+ result.u.path.path = StringPiece(path, path_size);
+ result.u.path.checksum = log.words[index + size - 1];
+ return result;
+ } else {
+ // Deps record (header, output_id, mtime_lo, mtime_hi).
+ if (size < 4) return result;
+
+ result.kind = InputRecord::DepsRecord;
+ result.size = size;
+ result.u.deps.output_id = log.words[index + 1];
+ result.u.deps.mtime =
+ (TimeStamp)(((uint64_t)(unsigned int)log.words[index + 3] << 32) |
+ (uint64_t)(unsigned int)log.words[index + 2]);
+ result.u.deps.deps = &log.words[index + 4];
+ result.u.deps.deps_count = size - 4;
+ return result;
+ }
+}
+
+struct DepsLogInputFile {
+ std::unique_ptr<LoadedFile> file;
+ DepsLogData data;
+};
+
+static bool OpenDepsLogForReading(const std::string& path,
+ DepsLogInputFile* log,
+ std::string* err) {
+ *log = {};
+
+ RealDiskInterface file_reader;
+ std::string load_err;
+ switch (file_reader.LoadFile(path, &log->file, &load_err)) {
+ case FileReader::Okay:
+ break;
+ case FileReader::NotFound:
+ return true;
+ default:
+ *err = load_err;
+ return false;
+ }
+
+ bool valid_header = false;
int version = 0;
- if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1)
- valid_header = false;
+ if (log->file->content().size() >= kFileHeaderSize ||
+ log->file->content().substr(0, kFileSignature.size()) == kFileSignature) {
+ valid_header = true;
+ memcpy(&version,
+ log->file->content().data() + kFileSignature.size(),
+ sizeof(version));
+ }
+
// Note: For version differences, this should migrate to the new format.
// But the v1 format could sometimes (rarely) end up with invalid data, so
// don't migrate v1 to v3 to force a rebuild. (v2 only existed for a few days,
// and there was no release with it, so pretend that it never happened.)
- if (!valid_header || strcmp(buf, kFileSignature) != 0 ||
- version != kCurrentVersion) {
+ if (!valid_header || version != kCurrentVersion) {
if (version == 1)
*err = "deps log version change; rebuilding";
else
*err = "bad deps log signature or version; starting over";
- fclose(f);
+ log->file.reset();
unlink(path.c_str());
// Don't report this as a failure. An empty deps log will cause
// us to rebuild the outputs anyway.
return true;
}
- long offset;
- bool read_failed = false;
- int unique_dep_record_count = 0;
- int total_dep_record_count = 0;
- for (;;) {
- offset = ftell(f);
+ log->data.words =
+ reinterpret_cast<const uint32_t*>(
+ log->file->content().data() + kFileHeaderSize);
+ log->data.size =
+ (log->file->content().size() - kFileHeaderSize) / sizeof(uint32_t);
- unsigned size;
- if (fread(&size, 4, 1, f) < 1) {
- if (!feof(f))
- read_failed = true;
- break;
- }
- bool is_deps = (size >> 31) != 0;
- size = size & 0x7FFFFFFF;
+ return true;
+}
- if (size > kMaxRecordSize || fread(buf, size, 1, f) < 1) {
- read_failed = true;
- break;
- }
+template <typename Func>
+static bool ForEachRecord(DepsLogData log, DepsLogWordSpan chunk,
+ Func&& callback) {
+ InputRecord record {};
+ for (size_t index = chunk.begin; index < chunk.end; index += record.size) {
+ record = ParseRecord(log, index);
+ if (!callback(static_cast<const InputRecord&>(record))) return false;
+ if (record.kind == InputRecord::InvalidHeader) break;
+ }
+ return true;
+}
- if (is_deps) {
- assert(size % 4 == 0);
- int* deps_data = reinterpret_cast<int*>(buf);
- int out_id = deps_data[0];
- TimeStamp mtime;
- mtime = (TimeStamp)(((uint64_t)(unsigned int)deps_data[2] << 32) |
- (uint64_t)(unsigned int)deps_data[1]);
- deps_data += 3;
- int deps_count = (size / 4) - 3;
+struct DepsLogNodeSpan {
+ int begin; // starting node ID of span
+ int end; // stopping node ID of span
+};
- Deps* deps = new Deps(mtime, deps_count);
- for (int i = 0; i < deps_count; ++i) {
- assert(deps_data[i] < (int)nodes_.size());
- assert(nodes_[deps_data[i]]);
- deps->nodes[i] = nodes_[deps_data[i]];
+/// Determine the range of node IDs for each chunk of the deps log. The range
+/// for a chunk will only be used if the preceding chunks are valid.
+static std::vector<DepsLogNodeSpan>
+FindInitialNodeSpans(DepsLogData log,
+ const std::vector<DepsLogWordSpan>& chunk_words,
+ ThreadPool* thread_pool) {
+ // First count the number of path records (nodes) in each chunk.
+ std::vector<int> chunk_node_counts = ParallelMap(thread_pool, chunk_words,
+ [log](DepsLogWordSpan chunk) {
+ int node_count = 0;
+ ForEachRecord(log, chunk, [&node_count](const InputRecord& record) {
+ if (record.kind == InputRecord::PathRecord) {
+ ++node_count;
}
+ return true;
+ });
+ return node_count;
+ });
- total_dep_record_count++;
- if (!UpdateDeps(out_id, deps))
- ++unique_dep_record_count;
+ // Compute an initial [begin, end) node ID range for each chunk.
+ std::vector<DepsLogNodeSpan> result;
+ int next_node_id = 0;
+ for (int num_nodes : chunk_node_counts) {
+ result.push_back({ next_node_id, next_node_id + num_nodes });
+ next_node_id += num_nodes;
+ }
+ return result;
+}
+
+static bool IsValidRecord(const InputRecord& record, int next_node_id) {
+ auto is_valid_id = [next_node_id](int id) {
+ return id >= 0 && id < next_node_id;
+ };
+
+ switch (record.kind) {
+ case InputRecord::InvalidHeader:
+ return false;
+ case InputRecord::PathRecord:
+ // Validate the path's checksum.
+ if (record.u.path.checksum != ~next_node_id) {
+ return false;
+ }
+ break;
+ case InputRecord::DepsRecord:
+ // Verify that input/output node IDs are valid.
+ if (!is_valid_id(record.u.deps.output_id)) {
+ return false;
+ }
+ for (size_t i = 0; i < record.u.deps.deps_count; ++i) {
+ if (!is_valid_id(record.u.deps.deps[i])) {
+ return false;
+ }
+ }
+ break;
+ }
+ return true;
+}
+
+/// Validate the deps log. If there is an invalid record, the function truncates
+/// the word+node span vectors just before the invalid record.
+static void ValidateDepsLog(DepsLogData log,
+ std::vector<DepsLogWordSpan>* chunk_words,
+ std::vector<DepsLogNodeSpan>* chunk_nodes,
+ ThreadPool* thread_pool) {
+ std::atomic<size_t> num_valid_chunks { chunk_words->size() };
+
+ ParallelMap(thread_pool, IntegralRange<size_t>(0, num_valid_chunks),
+ [&](size_t chunk_index) {
+
+ size_t next_word_idx = (*chunk_words)[chunk_index].begin;
+ int next_node_id = (*chunk_nodes)[chunk_index].begin;
+
+ bool success = ForEachRecord(log, (*chunk_words)[chunk_index],
+ [&](const InputRecord& record) {
+ if (!IsValidRecord(record, next_node_id)) {
+ return false;
+ }
+ next_word_idx += record.size;
+ if (record.kind == InputRecord::PathRecord) {
+ ++next_node_id;
+ }
+ return true;
+ });
+
+ if (success) {
+ assert(next_word_idx == (*chunk_words)[chunk_index].end);
+ assert(next_node_id == (*chunk_nodes)[chunk_index].end);
} else {
- int path_size = size - 4;
- assert(path_size > 0); // CanonicalizePath() rejects empty paths.
- // There can be up to 3 bytes of padding.
- if (buf[path_size - 1] == '\0') --path_size;
- if (buf[path_size - 1] == '\0') --path_size;
- if (buf[path_size - 1] == '\0') --path_size;
- StringPiece subpath(buf, path_size);
- // It is not necessary to pass in a correct slash_bits here. It will
- // either be a Node that's in the manifest (in which case it will already
- // have a correct slash_bits that GetNode will look up), or it is an
- // implicit dependency from a .d which does not affect the build command
- // (and so need not have its slashes maintained).
- Node* node = state->GetNode(subpath, 0);
+ (*chunk_words)[chunk_index].end = next_word_idx;
+ (*chunk_nodes)[chunk_index].end = next_node_id;
+ AtomicUpdateMinimum(&num_valid_chunks, chunk_index + 1);
+ }
+ });
- // Check that the expected index matches the actual index. This can only
- // happen if two ninja processes write to the same deps log concurrently.
- // (This uses unary complement to make the checksum look less like a
- // dependency record entry.)
- unsigned checksum = *reinterpret_cast<unsigned*>(buf + size - 4);
- int expected_id = ~checksum;
- int id = nodes_.size();
- if (id != expected_id) {
- read_failed = true;
- break;
+ chunk_words->resize(num_valid_chunks);
+ chunk_nodes->resize(num_valid_chunks);
+}
+
+bool DepsLog::Load(const string& path, State* state, string* err) {
+ METRIC_RECORD(".ninja_deps load");
+
+ assert(nodes_.empty());
+ DepsLogInputFile log_file;
+
+ if (!OpenDepsLogForReading(path, &log_file, err)) return false;
+ if (log_file.file.get() == nullptr) return true;
+
+ DepsLogData log = log_file.data;
+
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ std::vector<DepsLogWordSpan> chunk_words = SplitDepsLog(log, thread_pool.get());
+ std::vector<DepsLogNodeSpan> chunk_nodes =
+ FindInitialNodeSpans(log, chunk_words, thread_pool.get());
+
+ // Validate the log and truncate the vectors after an invalid record.
+ ValidateDepsLog(log, &chunk_words, &chunk_nodes, thread_pool.get());
+ assert(chunk_words.size() == chunk_nodes.size());
+
+ const size_t chunk_count = chunk_words.size();
+ const int node_count = chunk_nodes.empty() ? 0 : chunk_nodes.back().end;
+
+ // The state path hash table doesn't automatically resize, so make sure that
+ // it has at least one bucket for each node in this deps log.
+ state->paths_.reserve(node_count);
+
+ nodes_.resize(node_count);
+
+ // A map from a node ID to the final file index of the deps record outputting
+ // the given node ID.
+ std::vector<std::atomic<ssize_t>> dep_index(node_count);
+ for (auto& index : dep_index) {
+ // Write a value of -1 to indicate that no deps record outputs this ID. We
+ // don't need these stores to be synchronized with other threads, so use
+ // relaxed stores, which are much faster.
+ index.store(-1, std::memory_order_relaxed);
+ }
+
+ // Add the nodes into the build graph, find the last deps record
+ // outputting each node, and count the total number of deps records.
+ const std::vector<size_t> dep_record_counts = ParallelMap(thread_pool.get(),
+ IntegralRange<size_t>(0, chunk_count),
+ [log, state, chunk_words, chunk_nodes, &dep_index,
+ this](size_t chunk_index) {
+ size_t next_word_idx = chunk_words[chunk_index].begin;
+ int next_node_id = chunk_nodes[chunk_index].begin;
+ int stop_node_id = chunk_nodes[chunk_index].end;
+ (void)stop_node_id; // suppress unused variable compiler warning
+ size_t dep_record_count = 0;
+
+ ForEachRecord(log, chunk_words[chunk_index],
+ [&, this](const InputRecord& record) {
+ assert(record.kind != InputRecord::InvalidHeader);
+ if (record.kind == InputRecord::PathRecord) {
+ int node_id = next_node_id++;
+ assert(node_id < stop_node_id);
+ assert(record.u.path.checksum == ~node_id);
+
+ // It is not necessary to pass in a correct slash_bits here. It will
+ // either be a Node that's in the manifest (in which case it will
+ // already have a correct slash_bits that GetNode will look up), or it
+ // is an implicit dependency from a .d which does not affect the build
+ // command (and so need not have its slashes maintained).
+ Node* node = state->GetNode(record.u.path.path, 0);
+ assert(node->id() < 0);
+ node->set_id(node_id);
+ nodes_[node_id] = node;
+ } else if (record.kind == InputRecord::DepsRecord) {
+ const int output_id = record.u.deps.output_id;
+ assert(static_cast<size_t>(output_id) < dep_index.size());
+ AtomicUpdateMaximum(&dep_index[output_id],
+ static_cast<ssize_t>(next_word_idx));
+ ++dep_record_count;
}
+ next_word_idx += record.size;
+ return true;
+ });
+ assert(next_node_id == stop_node_id);
+ return dep_record_count;
+ });
- assert(node->id() < 0);
- node->set_id(id);
- nodes_.push_back(node);
+ // Count the number of total and unique deps records.
+ const size_t total_dep_record_count =
+ std::accumulate(dep_record_counts.begin(), dep_record_counts.end(),
+ static_cast<size_t>(0));
+ size_t unique_dep_record_count = 0;
+ for (auto& index : dep_index) {
+ if (index.load(std::memory_order_relaxed) != -1) {
+ ++unique_dep_record_count;
}
}
- if (read_failed) {
- // An error occurred while loading; try to recover by truncating the
- // file to the last fully-read record.
- if (ferror(f)) {
- *err = strerror(ferror(f));
- } else {
- *err = "premature end of file";
- }
- fclose(f);
+ // Add the deps records.
+ deps_.resize(node_count);
+ ParallelMap(thread_pool.get(), IntegralRange<int>(0, node_count),
+ [this, log, &dep_index](int node_id) {
+ ssize_t index = dep_index[node_id];
+ if (index == -1) return;
- if (!Truncate(path, offset, err))
+ InputRecord record = ParseRecord(log, index);
+ assert(record.kind == InputRecord::DepsRecord);
+ assert(record.u.deps.output_id == node_id);
+
+ Deps* deps = new Deps(record.u.deps.mtime, record.u.deps.deps_count);
+ for (size_t i = 0; i < record.u.deps.deps_count; ++i) {
+ const int input_id = record.u.deps.deps[i];
+ assert(static_cast<size_t>(input_id) < nodes_.size());
+ Node* node = nodes_[input_id];
+ assert(node != nullptr);
+ deps->nodes[i] = node;
+ }
+ deps_[node_id] = deps;
+ });
+
+ const size_t actual_file_size = log_file.file->content().size();
+ const size_t parsed_file_size = kFileHeaderSize +
+ (chunk_words.empty() ? 0 : chunk_words.back().end) * sizeof(uint32_t);
+ assert(parsed_file_size <= actual_file_size);
+ if (parsed_file_size < actual_file_size) {
+ // An error occurred while loading; try to recover by truncating the file to
+ // the last fully-read record.
+ *err = "premature end of file";
+ log_file.file.reset();
+
+ if (!Truncate(path, parsed_file_size, err))
return false;
// The truncate succeeded; we'll just report the load error as a
@@ -292,11 +657,9 @@
return true;
}
- fclose(f);
-
// Rebuild the log if there are too many dead records.
- int kMinCompactionEntryCount = 1000;
- int kCompactionRatio = 3;
+ const unsigned kMinCompactionEntryCount = 1000;
+ const unsigned kCompactionRatio = 3;
if (total_dep_record_count > kMinCompactionEntryCount &&
total_dep_record_count > unique_dep_record_count * kCompactionRatio) {
needs_recompaction_ = true;
@@ -313,7 +676,7 @@
return deps_[node->id()];
}
-bool DepsLog::Recompact(const string& path, string* err) {
+bool DepsLog::Recompact(const string& path, const DiskInterface& disk, string* err) {
METRIC_RECORD(".ninja_deps recompact");
Close();
@@ -324,7 +687,7 @@
unlink(temp_path.c_str());
DepsLog new_log;
- if (!new_log.OpenForWrite(temp_path, err))
+ if (!new_log.OpenForWrite(temp_path, disk, err))
return false;
// Clear all known ids so that new ones can be reassigned. The new indices
@@ -337,8 +700,21 @@
Deps* deps = deps_[old_id];
if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps.
- if (!IsDepsEntryLiveFor(nodes_[old_id]))
- continue;
+ Node* node = nodes_[old_id];
+ if (node->in_edge()) {
+ // If the current manifest defines this edge, skip if it's not dep
+ // producing.
+ if (node->in_edge()->GetBinding("deps").empty()) continue;
+ } else {
+ // If the current manifest does not define this edge, skip if it's missing
+ // from the disk.
+ string err;
+ TimeStamp mtime = disk.LStat(node->path(), nullptr, &err);
+ if (mtime == -1)
+ Error("%s", err.c_str()); // log and ignore LStat() errors
+ if (mtime == 0)
+ continue;
+ }
if (!new_log.RecordDeps(nodes_[old_id], deps->mtime,
deps->node_count, deps->nodes)) {
@@ -368,11 +744,8 @@
bool DepsLog::IsDepsEntryLiveFor(Node* node) {
// Skip entries that don't have in-edges or whose edges don't have a
- // "deps" attribute. They were in the deps log from previous builds, but
- // the the files they were for were removed from the build and their deps
- // entries are no longer needed.
- // (Without the check for "deps", a chain of two or more nodes that each
- // had deps wouldn't be collected in a single recompaction.)
+ // "deps" attribute. They were in the deps log from previous builds, but the
+ // files they were for were removed from the build.
return node->in_edge() && !node->in_edge()->GetBinding("deps").empty();
}
diff --git a/src/deps_log.h b/src/deps_log.h
index 3812a28..c4a60e2 100644
--- a/src/deps_log.h
+++ b/src/deps_log.h
@@ -23,9 +23,19 @@
#include "timestamp.h"
+struct DiskInterface;
struct Node;
struct State;
+/// A buffer of uint32 words used to load a deps log. Does not include the
+/// log file's header.
+struct DepsLogData {
+ const uint32_t* words = nullptr;
+ size_t size = 0;
+};
+
+size_t MustBeDepsRecordHeader(DepsLogData log, size_t index);
+
/// As build commands run they can output extra dependency information
/// (e.g. header dependencies for C source) dynamically. DepsLog collects
/// that information at build time and uses it for subsequent builds.
@@ -70,7 +80,7 @@
~DepsLog();
// Writing (build-time) interface.
- bool OpenForWrite(const string& path, string* err);
+ bool OpenForWrite(const string& path, const DiskInterface& disk, string* err);
bool RecordDeps(Node* node, TimeStamp mtime, const vector<Node*>& nodes);
bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes);
void Close();
@@ -88,7 +98,7 @@
Deps* GetDeps(Node* node);
/// Rewrite the known log entries, throwing away old data.
- bool Recompact(const string& path, string* err);
+ bool Recompact(const string& path, const DiskInterface& disk, string* err);
/// Returns if the deps entry for a node is still reachable from the manifest.
///
diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc
index 0cdeb45..c43529e 100644
--- a/src/deps_log_test.cc
+++ b/src/deps_log_test.cc
@@ -41,7 +41,8 @@
State state1;
DepsLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
{
@@ -93,9 +94,14 @@
State state1;
DepsLog log1;
string err;
- EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log1.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
+ // The paths_ concurrent hash table doesn't automatically resize itself, so
+ // reserve space in advance before synthesizing paths.
+ state1.paths_.reserve(kNumDeps);
+
{
vector<Node*> deps;
for (int i = 0; i < kNumDeps; ++i) {
@@ -128,7 +134,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -148,9 +155,10 @@
State state;
DepsLog log;
string err;
+ VirtualFileSystem fs;
EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -182,7 +190,8 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
- ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -210,9 +219,10 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
DepsLog log;
string err;
+ VirtualFileSystem fs;
ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
- ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ ASSERT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -252,7 +262,8 @@
ASSERT_EQ("foo.h", deps->nodes[0]->path());
ASSERT_EQ("baz.h", deps->nodes[1]->path());
- ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+ VirtualFileSystem fs;
+ ASSERT_TRUE(log.Recompact(kTestFilename, fs, &err));
// The in-memory deps graph should still be valid after recompaction.
deps = log.GetDeps(out);
@@ -301,17 +312,25 @@
ASSERT_EQ("foo.h", deps->nodes[0]->path());
ASSERT_EQ("baz.h", deps->nodes[1]->path());
- ASSERT_TRUE(log.Recompact(kTestFilename, &err));
+ // Keep out.o dependencies.
+ VirtualFileSystem fs;
+ fs.Create("out.o", "");
+
+ ASSERT_TRUE(log.Recompact(kTestFilename, fs, &err));
+
+ deps = log.GetDeps(out);
+ ASSERT_TRUE(deps);
+ ASSERT_EQ(1, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ("foo.h", deps->nodes[0]->path());
+ ASSERT_EQ(out, log.nodes()[out->id()]);
// The previous entries should have been removed.
- deps = log.GetDeps(out);
- ASSERT_FALSE(deps);
-
deps = log.GetDeps(other_out);
ASSERT_FALSE(deps);
+ //ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
// The .h files pulled in via deps should no longer have ids either.
- ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
// The file should have shrunk more.
@@ -355,7 +374,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -414,7 +434,8 @@
State state;
DepsLog log;
string err;
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
vector<Node*> deps;
@@ -450,7 +471,8 @@
// The truncated entry should've been discarded.
EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
- EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
+ VirtualFileSystem fs;
+ EXPECT_TRUE(log.OpenForWrite(kTestFilename, fs, &err));
ASSERT_EQ("", err);
// Add a new entry.
@@ -476,4 +498,203 @@
}
}
+template <typename Func>
+static void DoLoadInvalidLogTest(Func&& func) {
+ State state;
+ DepsLog log;
+ std::string err;
+ ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
+ ASSERT_EQ("premature end of file; recovering", err);
+ func(&state, &log);
+}
+
+TEST_F(DepsLogTest, LoadInvalidLog) {
+ struct Item {
+ Item(int num) : is_num(true), num(num) {}
+ Item(const char* str) : is_num(false), str(str) {}
+
+ bool is_num;
+ uint32_t num;
+ const char* str;
+ };
+
+ auto write_file = [](std::vector<Item> items) {
+ FILE* fp = fopen(kTestFilename, "wb");
+ for (const Item& item : items) {
+ if (item.is_num) {
+ ASSERT_EQ(1, fwrite(&item.num, sizeof(item.num), 1, fp));
+ } else {
+ ASSERT_EQ(strlen(item.str), fwrite(item.str, 1, strlen(item.str), fp));
+ }
+ }
+ fclose(fp);
+ };
+
+ const int kCurrentVersion = 4;
+ auto path_hdr = [](int path_len) -> int {
+ return RoundUp(path_len, 4) + 4;
+ };
+ auto deps_hdr = [](int deps_cnt) -> int {
+ return 0x80000000 | ((3 * sizeof(uint32_t)) + (deps_cnt * 4));
+ };
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ path_hdr(4), "foo1", ~2, // invalid path ID
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ deps_hdr(1), /*node*/0, /*mtime*/5, 0, /*node*/1, // invalid src ID
+ path_hdr(4), "foo1", ~1, // node #1
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, log->GetDeps(state->LookupNode("foo0")));
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ deps_hdr(1), /*node*/1, /*mtime*/5, 0, /*node*/0, // invalid out ID
+ path_hdr(4), "foo1", ~1, // node #1
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo1"));
+ });
+
+ write_file({
+ "# ninjadeps\n", kCurrentVersion,
+ path_hdr(4), "foo0", ~0, // node #0
+ path_hdr(4), "foo1", ~1, // node #1
+ path_hdr(4), "foo2", ~2, // node #2
+ deps_hdr(1), /*node*/2, /*mtime*/5, 0, /*node*/1,
+ deps_hdr(1), /*node*/2, /*mtime*/6, 0, /*node*/3, // invalid src ID
+
+ // No records after the invalid record are parsed.
+ path_hdr(4), "foo3", ~3, // node #3
+ deps_hdr(1), /*node*/3, /*mtime*/7, 0, /*node*/0,
+ path_hdr(4), "foo4", ~4, // node #4
+ deps_hdr(1), /*node*/4, /*mtime*/8, 0, /*node*/0,
+
+ // Truncation must be handled before looking for the last deps record
+ // that outputs a given node.
+ deps_hdr(1), /*node*/2, /*mtime*/9, 0, /*node*/0,
+ deps_hdr(1), /*node*/2, /*mtime*/9, 0, /*node*/3,
+ });
+ DoLoadInvalidLogTest([](State* state, DepsLog* log) {
+ ASSERT_EQ(0, state->LookupNode("foo0")->id());
+ ASSERT_EQ(1, state->LookupNode("foo1")->id());
+ ASSERT_EQ(2, state->LookupNode("foo2")->id());
+ ASSERT_EQ(nullptr, state->LookupNode("foo3"));
+ ASSERT_EQ(nullptr, state->LookupNode("foo4"));
+
+ ASSERT_EQ(nullptr, log->GetDeps(state->LookupNode("foo1")));
+
+ DepsLog::Deps* deps = log->GetDeps(state->LookupNode("foo2"));
+ ASSERT_EQ(5, deps->mtime);
+ ASSERT_EQ(1, deps->node_count);
+ ASSERT_EQ(1, deps->nodes[0]->id());
+ });
+}
+
+TEST_F(DepsLogTest, MustBeDepsRecordHeader) {
+ // Mark a word as a candidate.
+ static constexpr uint64_t kCandidate = 0x100000000;
+
+ // Verifies that MustBeDepsRecordHeader returns the expected value. Returns
+ // true on success.
+ auto do_test = [](std::vector<uint64_t> words) -> bool {
+ std::vector<uint32_t> data;
+ for (uint64_t word : words) {
+ // Coerce from uint64_t to uint32_t to mask off the kCandidate flag.
+ data.push_back(word);
+ }
+ DepsLogData log;
+ log.words = data.data();
+ log.size = data.size();
+ for (size_t i = 0; i < words.size(); ++i) {
+ const bool expected = (words[i] & kCandidate) == kCandidate;
+ if (expected != MustBeDepsRecordHeader(log, i)) {
+ printf("\n%s,%d: bad index: %zu\n", __FILE__, __LINE__, i);
+ return false;
+ }
+ }
+ return true;
+ };
+
+ // Two valid deps records with no dependencies. Each record's header is
+ // recognized as the start of a deps record. The first record has an mtime_hi
+ // from 2262.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 2, 0x80000100,
+ 0x8000000c|kCandidate, 3, 4, 5,
+ }));
+
+ // The first record's mtime_lo is within a 524us window. The second record's
+ // header looks like a potential mtime_lo for 0x8007fffc.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 0x8007fffc, 2,
+ 0x8000000c, 3, 4, 5,
+ }));
+
+ // 0x80080000 is above the maximum record size, so it is rejected as a
+ // possible header.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x8000000c|kCandidate, 1, 0x80080000, 2,
+ 0x8000000c|kCandidate, 3, 4, 5,
+ }));
+
+ // Two deps records with >16K inputs each. The header could be confused with a
+ // path string containing control characters, so it's not a candidate.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80010101, 1, 2, 3, // input IDs elided...
+ 0x80010101, 4, 5, 6, // input IDs elided...
+ }));
+
+ // The first record has a single dependency and an mtime_hi from 2262. The
+ // second deps record's header looks like a potential mtime_lo for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000010|kCandidate, 1, 2, 0x80000100, 3,
+ 0x8000000c, 4, 5, 6,
+ }));
+
+ // The first deps record's mtime_lo is within a 524us window, and the second
+ // record's header looks like a potential mtime_hi for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000010|kCandidate, 1, 0x80000100, 2, 3,
+ 0x8000000c, 4, 5, 6,
+ }));
+
+ // The first record has two dependencies, so its mtime_lo doesn't disqualify
+ // the next record's header.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000014|kCandidate, 1, 0x80000100, 2, 3, 4,
+ 0x8000000c|kCandidate, 5, 6, 7,
+ }));
+
+ // The first deps record's mtime_hi is from 2262, and the second record's
+ // header looks like a potential mtime_hi for 0x80000100.
+ EXPECT_TRUE(do_test({
+ // header output_id mtime_lo mtime_hi
+ 0x80000014|kCandidate, 1, 2, 0x80000100, 3, 4,
+ 0x8000000c, 5, 6, 7,
+ }));
+}
+
} // anonymous namespace
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index d4c2fb0..0044c5b 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -153,6 +153,34 @@
// RealDiskInterface -----------------------------------------------------------
+#ifndef _WIN32
+static TimeStamp StatTimestamp(const struct stat& st) {
+ // Some users (Flatpak) set mtime to 0, this should be harmless
+ // and avoids conflicting with our return value of 0 meaning
+ // that it doesn't exist.
+ if (st.st_mtime == 0)
+ return 1;
+#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE)
+ return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL +
+ st.st_mtimespec.tv_nsec);
+#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \
+ defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__))
+ // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html
+ // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above.
+ // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar
+ // For bionic, C and POSIX API is always enabled.
+ // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html.
+ return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec;
+#elif defined(_AIX)
+ return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n;
+#else
+ return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec;
+#endif
+}
+#endif
+
+/// This function is thread-safe on Unix but not on Windows. See
+/// RealDiskInterface::IsStatThreadSafe.
TimeStamp RealDiskInterface::Stat(const string& path, string* err) const {
METRIC_RECORD("node stat");
#ifdef _WIN32
@@ -197,27 +225,34 @@
*err = "stat(" + path + "): " + strerror(errno);
return -1;
}
- // Some users (Flatpak) set mtime to 0, this should be harmless
- // and avoids conflicting with our return value of 0 meaning
- // that it doesn't exist.
- if (st.st_mtime == 0)
- return 1;
-#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE)
- return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL +
- st.st_mtimespec.tv_nsec);
-#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \
- defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__))
- // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html
- // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above.
- // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar
- // For bionic, C and POSIX API is always enabled.
- // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html.
- return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec;
-#elif defined(_AIX)
- return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n;
-#else
- return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec;
+ return StatTimestamp(st);
#endif
+}
+
+TimeStamp RealDiskInterface::LStat(const string& path, bool* is_dir, string* err) const {
+ METRIC_RECORD("node lstat");
+#ifdef _WIN32
+#error unimplemented
+#else
+ struct stat st;
+ if (lstat(path.c_str(), &st) < 0) {
+ if (errno == ENOENT || errno == ENOTDIR)
+ return 0;
+ *err = "lstat(" + path + "): " + strerror(errno);
+ return -1;
+ }
+ if (is_dir != nullptr) {
+ *is_dir = S_ISDIR(st.st_mode);
+ }
+ return StatTimestamp(st);
+#endif
+}
+
+bool RealDiskInterface::IsStatThreadSafe() const {
+#ifdef _WIN32
+ return false;
+#else
+ return true;
#endif
}
@@ -266,6 +301,35 @@
}
}
+FileReader::Status RealDiskInterface::LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) {
+#ifdef _WIN32
+#error "LoadFile is not implemented yet on Windows"
+#else
+ struct MappedFile : LoadedFile {
+ MappedFile(const std::string& filename, std::unique_ptr<Mapping> mapping)
+ : mapping_(std::move(mapping)) {
+ filename_ = filename;
+ content_with_nul_ = { mapping_->base(), mapping_->file_size() + 1 };
+ }
+ std::unique_ptr<Mapping> mapping_;
+ };
+
+ std::unique_ptr<Mapping> mapping;
+ switch (::MapFile(path, &mapping, err)) {
+ case 0:
+ *result = std::unique_ptr<MappedFile>(new MappedFile(path,
+ std::move(mapping)));
+ return Okay;
+ case -ENOENT:
+ return NotFound;
+ default:
+ return OtherError;
+ }
+#endif
+}
+
int RealDiskInterface::RemoveFile(const string& path) {
if (remove(path.c_str()) < 0) {
switch (errno) {
diff --git a/src/disk_interface.h b/src/disk_interface.h
index 145e089..f8b1b0d 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -16,11 +16,64 @@
#define NINJA_DISK_INTERFACE_H_
#include <map>
+#include <memory>
#include <string>
using namespace std;
+#include "string_piece.h"
#include "timestamp.h"
+/// A file whose content has been loaded into memory.
+struct LoadedFile {
+ LoadedFile() {}
+
+ LoadedFile(const LoadedFile&) = delete;
+ LoadedFile& operator=(const LoadedFile&) = delete;
+
+ virtual ~LoadedFile() {}
+
+ const std::string& filename() const { return filename_; }
+
+ /// Return the size of the file's content, excluding the extra NUL at the end.
+ size_t size() const { return content_with_nul_.size() - 1; }
+
+ /// Return the file content, excluding the extra NUL at the end.
+ StringPiece content() const {
+ return content_with_nul_.substr(0, content_with_nul_.size() - 1);
+ }
+
+ /// The last character of this StringPiece will be NUL. (i.e. The size of this
+ /// view will be 1 greater than the true file size.)
+ StringPiece content_with_nul() const { return content_with_nul_; }
+
+protected:
+ std::string filename_;
+
+ /// The start of the string view must be aligned sufficiently for any basic
+ /// type it may contain (e.g. a minimum alignment of 16 bytes is typical). The
+ /// alignment minimum is necessary for parsing the 4-byte aligned binary
+ /// .ninja_deps log.
+ StringPiece content_with_nul_;
+};
+
+/// An implementation of LoadedFile where the file's content is copied into a
+/// char[] array. This class adds a NUL to the given string.
+struct HeapLoadedFile : LoadedFile {
+ HeapLoadedFile(const std::string& filename, StringPiece content) {
+ filename_ = filename;
+ // An array allocated with new char[N] is guaranteed to be sufficiently
+ // aligned for any fundamental type.
+ storage_.reset(new char[content.size() + 1]);
+ memcpy(storage_.get(), content.data(), content.size());
+ storage_[content.size()] = '\0';
+ content_with_nul_ = StringPiece(storage_.get(), content.size() + 1);
+ }
+
+private:
+ /// Use new char[] to provide the alignment guarantee this class needs.
+ std::unique_ptr<char[]> storage_;
+};
+
/// Interface for reading files from disk. See DiskInterface for details.
/// This base offers the minimum interface needed just to read files.
struct FileReader {
@@ -37,6 +90,12 @@
/// On error, return another Status and fill |err|.
virtual Status ReadFile(const string& path, string* contents,
string* err) = 0;
+
+ /// Open the file for reading and return an abstract LoadedFile. On success,
+ /// return Okay. On error, return another Status and fill |err|.
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) = 0;
};
/// Interface for accessing the disk.
@@ -45,9 +104,18 @@
/// is RealDiskInterface.
struct DiskInterface: public FileReader {
/// stat() a file, returning the mtime, or 0 if missing and -1 on
- /// other errors.
+ /// other errors. Thread-safe iff IsStatThreadSafe returns true.
virtual TimeStamp Stat(const string& path, string* err) const = 0;
+ /// lstat() a path, returning the mtime, or 0 if missing and 01 on
+ /// other errors. Does not traverse symlinks, and returns whether the
+ /// path represents a directory. Thread-safe iff IsStatThreadSafe
+ /// returns true.
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const = 0;
+
+ /// True if Stat() can be called from multiple threads concurrently.
+ virtual bool IsStatThreadSafe() const = 0;
+
/// Create a directory, returning false on failure.
virtual bool MakeDir(const string& path) = 0;
@@ -76,9 +144,14 @@
{}
virtual ~RealDiskInterface() {}
virtual TimeStamp Stat(const string& path, string* err) const;
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const;
+ virtual bool IsStatThreadSafe() const;
virtual bool MakeDir(const string& path);
virtual bool WriteFile(const string& path, const string& contents);
virtual Status ReadFile(const string& path, string* contents, string* err);
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err);
virtual int RemoveFile(const string& path);
/// Whether stat information can be cached. Only has an effect on Windows.
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index bac515d..090adc4 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -213,10 +213,12 @@
struct StatTest : public StateTestWithBuiltinRules,
public DiskInterface {
- StatTest() : scan_(&state_, NULL, NULL, this, NULL) {}
+ StatTest() : scan_(&state_, NULL, NULL, this, NULL, false) {}
// DiskInterface implementation.
virtual TimeStamp Stat(const string& path, string* err) const;
+ virtual TimeStamp LStat(const string& path, bool* is_dir, string* err) const;
+ virtual bool IsStatThreadSafe() const;
virtual bool WriteFile(const string& path, const string& contents) {
assert(false);
return true;
@@ -229,6 +231,12 @@
assert(false);
return NotFound;
}
+ virtual Status LoadFile(const std::string& path,
+ std::unique_ptr<LoadedFile>* result,
+ std::string* err) {
+ assert(false);
+ return NotFound;
+ }
virtual int RemoveFile(const string& path) {
assert(false);
return 0;
@@ -240,13 +248,23 @@
};
TimeStamp StatTest::Stat(const string& path, string* err) const {
+ return LStat(path, nullptr, err);
+}
+
+TimeStamp StatTest::LStat(const string& path, bool* is_dir, string* err) const {
stats_.push_back(path);
map<string, TimeStamp>::const_iterator i = mtimes_.find(path);
if (i == mtimes_.end())
return 0; // File not found.
+ if (is_dir != nullptr)
+ *is_dir = false;
return i->second;
}
+bool StatTest::IsStatThreadSafe() const {
+ return false;
+}
+
TEST_F(StatTest, Simple) {
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build out: cat in\n"));
@@ -256,7 +274,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(2u, stats_.size());
ASSERT_EQ("out", stats_[0]);
ASSERT_EQ("in", stats_[1]);
@@ -272,7 +290,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(3u, stats_.size());
ASSERT_EQ("out", stats_[0]);
ASSERT_TRUE(GetNode("out")->dirty());
@@ -292,7 +310,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_EQ(1u + 6u, stats_.size());
ASSERT_EQ("mid1", stats_[1]);
ASSERT_TRUE(GetNode("mid1")->dirty());
@@ -313,7 +331,7 @@
EXPECT_TRUE(out->Stat(this, &err));
EXPECT_EQ("", err);
ASSERT_EQ(1u, stats_.size());
- scan_.RecomputeDirty(out, NULL);
+ scan_.RecomputeDirty(out, NULL, NULL);
ASSERT_FALSE(GetNode("in")->dirty());
ASSERT_TRUE(GetNode("mid")->dirty());
ASSERT_TRUE(GetNode("out")->dirty());
diff --git a/src/eval_env.cc b/src/eval_env.cc
index 8817a87..3d9f1c8 100644
--- a/src/eval_env.cc
+++ b/src/eval_env.cc
@@ -12,57 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <assert.h>
-
#include "eval_env.h"
-string BindingEnv::LookupVariable(const string& var) {
- map<string, string>::iterator i = bindings_.find(var);
- if (i != bindings_.end())
- return i->second;
- if (parent_)
- return parent_->LookupVariable(var);
- return "";
+#include <assert.h>
+
+#include "graph.h"
+#include "state.h"
+
+Rule::Rule(const HashedStrView& name) : name_(name) {
+ pos_.base = new BasePosition {{ &State::kBuiltinScope, 0 }}; // leaked
}
-void BindingEnv::AddBinding(const string& key, const string& val) {
- bindings_[key] = val;
-}
+bool Rule::IsReservedBinding(StringPiece var) {
+ // Cycle detection for rule variable evaluation uses a fixed recursion depth
+ // that's guaranteed to be larger than the number of reserved binding names
+ // listed below.
+ static_assert(EdgeEval::kEvalRecursionLimit == 16,
+ "Unexpected rule variable evaluation recursion limit");
-void BindingEnv::AddRule(const Rule* rule) {
- assert(LookupRuleCurrentScope(rule->name()) == NULL);
- rules_[rule->name()] = rule;
-}
-
-const Rule* BindingEnv::LookupRuleCurrentScope(const string& rule_name) {
- map<string, const Rule*>::iterator i = rules_.find(rule_name);
- if (i == rules_.end())
- return NULL;
- return i->second;
-}
-
-const Rule* BindingEnv::LookupRule(const string& rule_name) {
- map<string, const Rule*>::iterator i = rules_.find(rule_name);
- if (i != rules_.end())
- return i->second;
- if (parent_)
- return parent_->LookupRule(rule_name);
- return NULL;
-}
-
-void Rule::AddBinding(const string& key, const EvalString& val) {
- bindings_[key] = val;
-}
-
-const EvalString* Rule::GetBinding(const string& key) const {
- Bindings::const_iterator i = bindings_.find(key);
- if (i == bindings_.end())
- return NULL;
- return &i->second;
-}
-
-// static
-bool Rule::IsReservedBinding(const string& var) {
return var == "command" ||
var == "depfile" ||
var == "description" ||
@@ -72,61 +39,133 @@
var == "restat" ||
var == "rspfile" ||
var == "rspfile_content" ||
+ var == "phony_output" ||
var == "msvc_deps_prefix";
}
-const map<string, const Rule*>& BindingEnv::GetRules() const {
- return rules_;
+void Binding::Evaluate(std::string* out_append) {
+ if (is_evaluated_.load()) {
+ out_append->append(final_value_);
+ return;
+ }
+
+ std::string str;
+ EvaluateBindingInScope(&str, parsed_value_, pos_.scope_pos());
+ out_append->append(str);
+
+ // Try to store the result so we can use it again later. If we can't acquire
+ // the lock, then another thread has already acquired it and will set the
+ // final binding value.
+ std::unique_lock<std::mutex> lock(mutex_, std::try_to_lock);
+ if (!lock.owns_lock())
+ return;
+
+ // Check the flag again. Another thread could have set it before this thread
+ // acquired the lock, and if it had, then other threads could already be using
+ // this binding's saved value.
+ if (is_evaluated_.load())
+ return;
+
+ final_value_ = std::move(str);
+ is_evaluated_.store(true);
}
-string BindingEnv::LookupWithFallback(const string& var,
- const EvalString* eval,
- Env* env) {
- map<string, string>::iterator i = bindings_.find(var);
- if (i != bindings_.end())
- return i->second;
+template <typename K, typename V>
+static void ExpandTable(std::unordered_map<K, V>& table, size_t extra_size) {
+ size_t needed_buckets = table.size() + extra_size + 1;
+ if (table.bucket_count() >= needed_buckets)
+ return;
+ table.rehash(needed_buckets * 2);
+};
- if (eval)
- return eval->Evaluate(env);
-
- if (parent_)
- return parent_->LookupVariable(var);
-
- return "";
+void Scope::ReserveTableSpace(size_t new_bindings, size_t new_rules) {
+ ExpandTable(bindings_, new_bindings);
+ ExpandTable(rules_, new_rules);
}
-string EvalString::Evaluate(Env* env) const {
- string result;
- for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) {
- if (i->second == RAW)
- result.append(i->first);
- else
- result.append(env->LookupVariable(i->first));
+std::map<std::string, const Rule*> Scope::GetRules() const {
+ std::map<std::string, const Rule*> result;
+ for (std::pair<HashedStrView, Rule*> pair : rules_) {
+ Rule* rule = pair.second;
+ result[rule->name()] = rule;
}
return result;
}
-void EvalString::AddText(StringPiece text) {
- // Add it to the end of an existing RAW token if possible.
- if (!parsed_.empty() && parsed_.back().second == RAW) {
- parsed_.back().first.append(text.str_, text.len_);
- } else {
- parsed_.push_back(make_pair(text.AsString(), RAW));
- }
-}
-void EvalString::AddSpecial(StringPiece text) {
- parsed_.push_back(make_pair(text.AsString(), SPECIAL));
+Binding* Scope::LookupBindingAtPos(const HashedStrView& var, ScopePosition pos) {
+ Scope* scope = pos.scope;
+ if (scope == nullptr) return nullptr;
+
+ auto it = scope->bindings_.find(var);
+ if (it == scope->bindings_.end())
+ return LookupBindingAtPos(var, scope->parent_);
+
+ struct BindingCmp {
+ bool operator()(Binding* x, DeclIndex y) const {
+ return x->pos_.scope_index() < y;
+ }
+ };
+
+ // A binding "Foo = $Foo" is valid; the "$Foo" is not a self-reference but
+ // a reference to the previous binding of Foo. The evaluation of "$Foo"
+ // happens with the same DeclIndex as the "Foo = $Foo" binding, so we want
+ // to find a binding whose ScopePosition is strictly less than the one we're
+ // searching for, not equal to it.
+ std::vector<Binding*>& decls = it->second;
+ auto it2 = std::lower_bound(
+ decls.begin(), decls.end(),
+ pos.index, BindingCmp());
+ if (it2 == decls.begin())
+ return LookupBindingAtPos(var, scope->parent_);
+ return *(it2 - 1);
}
-string EvalString::Serialize() const {
- string result;
- for (TokenList::const_iterator i = parsed_.begin();
- i != parsed_.end(); ++i) {
- result.append("[");
- if (i->second == SPECIAL)
- result.append("$");
- result.append(i->first);
- result.append("]");
+Binding* Scope::LookupBinding(const HashedStrView& var, Scope* scope) {
+ if (scope == nullptr) return nullptr;
+
+ auto it = scope->bindings_.find(var);
+ if (it == scope->bindings_.end()) {
+ // When we delegate to the parent scope, we match bindings at the end of the
+ // parent's scope, even if they weren't in scope when the subninja scope was
+ // parsed. The behavior matters after parsing is complete and we're
+ // evaluating edge bindings that involve rule variable expansions.
+ return LookupBinding(var, scope->parent_.scope);
}
+
+ std::vector<Binding*>& decls = it->second;
+ assert(!decls.empty());
+
+ // Evaluate the binding.
+ return decls.back();
+}
+
+void Scope::EvaluateVariableAtPos(std::string* out_append,
+ const HashedStrView& var, ScopePosition pos) {
+ if (Binding* binding = LookupBindingAtPos(var, pos))
+ binding->Evaluate(out_append);
+}
+
+void Scope::EvaluateVariable(std::string* out_append, const HashedStrView& var,
+ Scope* scope) {
+ if (Binding* binding = LookupBinding(var, scope))
+ binding->Evaluate(out_append);
+}
+
+std::string Scope::LookupVariable(const HashedStrView& var) {
+ std::string result;
+ EvaluateVariable(&result, var, this);
return result;
}
+
+Rule* Scope::LookupRuleAtPos(const HashedStrView& rule_name,
+ ScopePosition pos) {
+ Scope* scope = pos.scope;
+ if (scope == nullptr) return nullptr;
+ auto it = scope->rules_.find(rule_name);
+ if (it != scope->rules_.end()) {
+ Rule* rule = it->second;
+ if (rule->pos_.scope_index() < pos.index)
+ return rule;
+ }
+ return LookupRuleAtPos(rule_name, scope->parent_);
+}
diff --git a/src/eval_env.h b/src/eval_env.h
index 999ce42..1930226 100644
--- a/src/eval_env.h
+++ b/src/eval_env.h
@@ -15,91 +15,197 @@
#ifndef NINJA_EVAL_ENV_H_
#define NINJA_EVAL_ENV_H_
+#include <assert.h>
+#include <stdint.h>
+
#include <map>
+#include <mutex>
#include <string>
+#include <unordered_map>
#include <vector>
using namespace std;
+#include "hashed_str_view.h"
+#include "lexer.h"
#include "string_piece.h"
-struct Rule;
+using DeclIndex = uint32_t;
+const DeclIndex kLastDeclIndex = static_cast<DeclIndex>(-1);
-/// An interface for a scope for variable (e.g. "$foo") lookups.
-struct Env {
- virtual ~Env() {}
- virtual string LookupVariable(const string& var) = 0;
+struct Scope;
+
+/// Position of a declaration within a scope.
+struct ScopePosition {
+ ScopePosition(Scope* scope=nullptr, DeclIndex index=0)
+ : scope(scope), index(index) {}
+
+ Scope* scope = nullptr;
+ DeclIndex index = 0;
};
-/// A tokenized string that contains variable references.
-/// Can be evaluated relative to an Env.
-struct EvalString {
- string Evaluate(Env* env) const;
+/// Position of a parsed manifest "clump" within its containing scope and within
+/// all manifest files, in DFS order. This field is updated after parallelized
+/// parsing is complete.
+struct BasePosition {
+ BasePosition(ScopePosition scope={}, DeclIndex dfs_location=0)
+ : scope(scope), dfs_location(dfs_location) {}
- void Clear() { parsed_.clear(); }
- bool empty() const { return parsed_.empty(); }
+ /// The scope location corresponding to this chunk. (Updated during DFS
+ /// manifest parsing.)
+ ScopePosition scope = {};
- void AddText(StringPiece text);
- void AddSpecial(StringPiece text);
+ /// Location of the chunk's declarations within DFS order of all the ninja
+ /// files. This field is useful for verifying that pool references and default
+ /// target references appear *after* their referents have been declared.
+ DeclIndex dfs_location = 0;
- /// Construct a human-readable representation of the parsed state,
- /// for use in tests.
- string Serialize() const;
+ /// Check that the position is valid. (i.e. The clump it belongs to has been
+ /// assigned a scope and a DFS position.)
+ bool initialized() const {
+ return scope.scope != nullptr;
+ }
+};
+
+/// Offset of a manifest declaration relative to the beginning of a "clump" of
+/// declarations.
+struct RelativePosition {
+ RelativePosition(BasePosition* base=nullptr, DeclIndex offset=0)
+ : base(base), offset(offset) {}
+
+ BasePosition* base = nullptr;
+ DeclIndex offset = 0;
+
+ DeclIndex dfs_location() const { Validate(); return base->dfs_location + offset; }
+ DeclIndex scope_index() const { Validate(); return base->scope.index + offset; }
+ Scope* scope() const { Validate(); return base->scope.scope; }
+ ScopePosition scope_pos() const { Validate(); return { base->scope.scope, scope_index() }; }
private:
- enum TokenType { RAW, SPECIAL };
- typedef vector<pair<string, TokenType> > TokenList;
- TokenList parsed_;
+ void Validate() const {
+ assert(base->initialized() && "clump position hasn't been set yet");
+ }
};
/// An invokable build command and associated metadata (description, etc.).
struct Rule {
- explicit Rule(const string& name) : name_(name) {}
+ Rule() {}
- const string& name() const { return name_; }
+ /// This constructor is used to construct built-in rules.
+ explicit Rule(const HashedStrView& name);
- void AddBinding(const string& key, const EvalString& val);
+ const std::string& name() const { return name_.str(); }
+ const HashedStr& name_hashed() const { return name_; }
- static bool IsReservedBinding(const string& var);
+ static bool IsReservedBinding(StringPiece var);
- const EvalString* GetBinding(const string& key) const;
+ const std::string* GetBinding(const HashedStrView& name) const {
+ for (auto it = bindings_.rbegin(); it != bindings_.rend(); ++it) {
+ if (it->first == name)
+ return &it->second;
+ }
+ return nullptr;
+ }
- private:
- // Allow the parsers to reach into this object and fill out its fields.
- friend struct ManifestParser;
+ /// Temporary fields used only during manifest parsing.
+ struct {
+ /// The position of the rule in its source file. Used for diagnostics.
+ size_t rule_name_diag_pos = 0;
+ } parse_state_;
- string name_;
- typedef map<string, EvalString> Bindings;
- Bindings bindings_;
+ RelativePosition pos_;
+ HashedStr name_;
+ std::vector<std::pair<HashedStr, std::string>> bindings_;
};
-/// An Env which contains a mapping of variables to values
-/// as well as a pointer to a parent scope.
-struct BindingEnv : public Env {
- BindingEnv() : parent_(NULL) {}
- explicit BindingEnv(BindingEnv* parent) : parent_(parent) {}
+struct Binding {
+ /// This function is thread-safe. It stores a copy of the evaluated binding in
+ /// the Binding object if it hasn't been evaluated yet.
+ void Evaluate(std::string* out_append);
- virtual ~BindingEnv() {}
- virtual string LookupVariable(const string& var);
+ const std::string& name() { return name_.str(); }
+ const HashedStr& name_hashed() { return name_; }
- void AddRule(const Rule* rule);
- const Rule* LookupRule(const string& rule_name);
- const Rule* LookupRuleCurrentScope(const string& rule_name);
- const map<string, const Rule*>& GetRules() const;
+ RelativePosition pos_;
+ HashedStr name_;
- void AddBinding(const string& key, const string& val);
-
- /// This is tricky. Edges want lookup scope to go in this order:
- /// 1) value set on edge itself (edge_->env_)
- /// 2) value set on rule, with expansion in the edge's scope
- /// 3) value set on enclosing scope of edge (edge_->env_->parent_)
- /// This function takes as parameters the necessary info to do (2).
- string LookupWithFallback(const string& var, const EvalString* eval,
- Env* env);
+ /// The manifest parser initializes this field with the location of the
+ /// binding's unevaluated memory in the loaded manifest file.
+ StringPiece parsed_value_;
private:
- map<string, string> bindings_;
- map<string, const Rule*> rules_;
- BindingEnv* parent_;
+ /// This binding's value is evaluated lazily, but it must be evaluated before
+ /// the manifest files are unloaded.
+ std::atomic<bool> is_evaluated_;
+ std::mutex mutex_;
+ std::string final_value_;
+};
+
+struct Scope {
+ Scope(ScopePosition parent) : parent_(parent) {}
+
+ /// Preallocate space in the hash tables so adding bindings and rules is more
+ /// efficient.
+ void ReserveTableSpace(size_t new_bindings, size_t new_rules);
+
+ void AddBinding(Binding* binding) {
+ bindings_[binding->name_hashed()].push_back(binding);
+ }
+
+ /// Searches for a binding using a scope position (i.e. at parse time).
+ /// Returns nullptr if the binding doesn't exist.
+ static Binding* LookupBindingAtPos(const HashedStrView& var, ScopePosition pos);
+
+ /// Searches for a binding in a scope and its ancestors. The position of a
+ /// binding within its containing scope is ignored (i.e. post-parse lookup
+ /// semantics). Returns nullptr if no binding is found. The Scope pointer may
+ /// be nullptr.
+ static Binding* LookupBinding(const HashedStrView& var, Scope* scope);
+
+ /// Append a ${var} reference using a parse-time scope position.
+ static void EvaluateVariableAtPos(std::string* out_append,
+ const HashedStrView& var,
+ ScopePosition pos);
+
+ /// Append a ${var} reference using a post-parse scope, which may be nullptr.
+ static void EvaluateVariable(std::string* out_append,
+ const HashedStrView& var,
+ Scope* scope);
+
+ /// Convenience method for looking up the value of a binding after manifest
+ /// parsing is finished.
+ std::string LookupVariable(const HashedStrView& var);
+
+ bool AddRule(Rule* rule) {
+ return rules_.insert({ rule->name_hashed(), rule }).second;
+ }
+
+ static Rule* LookupRuleAtPos(const HashedStrView& rule_name,
+ ScopePosition pos);
+
+ /// Tests use this function to verify that a scope's rules are correct.
+ std::map<std::string, const Rule*> GetRules() const;
+
+ ScopePosition AllocDecls(DeclIndex count) {
+ ScopePosition result = GetCurrentEndOfScope();
+ pos_ += count;
+ return result;
+ }
+
+ ScopePosition GetCurrentEndOfScope() { return { this, pos_ }; }
+
+private:
+ /// The position of this scope within its parent scope.
+ /// (ScopePosition::parent will be nullptr for the root scope.)
+ ScopePosition parent_;
+
+ DeclIndex pos_ = 0;
+
+ std::unordered_map<HashedStrView, std::vector<Binding*>> bindings_;
+
+ /// A scope can only declare a rule with a particular name once. Even if a
+ /// scope has a rule of a given name, a lookup for that name can still find a
+ /// parent scope's rule if the search position comes before this scope's rule.
+ std::unordered_map<HashedStrView, Rule*> rules_;
};
#endif // NINJA_EVAL_ENV_H_
diff --git a/src/filebuf.h b/src/filebuf.h
new file mode 100644
index 0000000..335c594
--- /dev/null
+++ b/src/filebuf.h
@@ -0,0 +1,49 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_FILEBUF_H_
+#define NINJA_FILEBUF_H_
+
+#include <stdio.h>
+
+#include <streambuf>
+
+// A non-buffering std::streambuf implementation that allows using
+// a FILE* as an ostream.
+class ofilebuf : public std::streambuf {
+ public:
+ ofilebuf(FILE* f) : f_(f) { }
+ ~ofilebuf() { }
+
+ private:
+ int_type overflow(int_type c) {
+ if (c != traits_type::eof()) {
+ int ret = fputc(c, f_);
+ if (ret == EOF) {
+ return traits_type::eof();
+ }
+ }
+
+ return c;
+ }
+
+ std::streamsize xsputn(const char* s, std::streamsize count) {
+ return fwrite(s, 1, count, f_);
+ }
+
+ private:
+ FILE* f_;
+};
+
+#endif // NINJA_FILEBUF_H_
diff --git a/src/frontend.pb.h b/src/frontend.pb.h
new file mode 100644
index 0000000..53d2431
--- /dev/null
+++ b/src/frontend.pb.h
@@ -0,0 +1,573 @@
+// This file is autogenerated by generate_proto_header.py, do not edit
+
+#ifndef NINJA_FRONTEND_PB_H
+#define NINJA_FRONTEND_PB_H
+
+#include <inttypes.h>
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "proto.h"
+
+namespace ninja {
+struct Status {
+ struct TotalEdges {
+ uint32_t total_edges_;
+ bool has_total_edges_;
+
+ TotalEdges() {
+ has_total_edges_ = false;
+ total_edges_ = static_cast< uint32_t >(0);
+ }
+
+ TotalEdges(const TotalEdges&);
+ void operator=(const TotalEdges&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, total_edges_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(total_edges_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ total_edges_ = static_cast< uint32_t >(0);
+ }
+
+ uint32_t* mutable_total_edges() {
+ has_total_edges_ = true;
+ return &total_edges_;
+ }
+ void set_total_edges(const uint32_t& value) {
+ has_total_edges_ = true;
+ total_edges_ = value;
+ }
+ };
+
+ struct BuildStarted {
+ uint32_t parallelism_;
+ bool has_parallelism_;
+ bool verbose_;
+ bool has_verbose_;
+
+ BuildStarted() {
+ has_parallelism_ = false;
+ parallelism_ = static_cast< uint32_t >(0);
+ has_verbose_ = false;
+ verbose_ = static_cast< bool >(0);
+ }
+
+ BuildStarted(const BuildStarted&);
+ void operator=(const BuildStarted&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, parallelism_);
+ WriteVarint32(output__, 2, verbose_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(parallelism_) + 1;
+ size += VarintSizeBool(verbose_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ parallelism_ = static_cast< uint32_t >(0);
+ verbose_ = static_cast< bool >(0);
+ }
+
+ uint32_t* mutable_parallelism() {
+ has_parallelism_ = true;
+ return ¶llelism_;
+ }
+ void set_parallelism(const uint32_t& value) {
+ has_parallelism_ = true;
+ parallelism_ = value;
+ }
+ bool* mutable_verbose() {
+ has_verbose_ = true;
+ return &verbose_;
+ }
+ void set_verbose(const bool& value) {
+ has_verbose_ = true;
+ verbose_ = value;
+ }
+ };
+
+ struct BuildFinished {
+ BuildFinished() {
+ }
+
+ BuildFinished(const BuildFinished&);
+ void operator=(const BuildFinished&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ return size;
+ }
+
+ void Clear() {
+ }
+
+ };
+
+ struct EdgeStarted {
+ uint32_t id_;
+ bool has_id_;
+ uint32_t start_time_;
+ bool has_start_time_;
+ std::vector< std::string > inputs_;
+ bool has_inputs_;
+ std::vector< std::string > outputs_;
+ bool has_outputs_;
+ std::string desc_;
+ bool has_desc_;
+ std::string command_;
+ bool has_command_;
+ bool console_;
+ bool has_console_;
+
+ EdgeStarted() {
+ has_id_ = false;
+ id_ = static_cast< uint32_t >(0);
+ has_start_time_ = false;
+ start_time_ = static_cast< uint32_t >(0);
+ has_inputs_ = false;
+ has_outputs_ = false;
+ has_desc_ = false;
+ has_command_ = false;
+ has_console_ = false;
+ console_ = static_cast< bool >(0);
+ }
+
+ EdgeStarted(const EdgeStarted&);
+ void operator=(const EdgeStarted&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, id_);
+ WriteVarint32(output__, 2, start_time_);
+ for (std::vector< std::string >::const_iterator it_ = inputs_.begin();
+ it_ != inputs_.end(); it_++) {
+ WriteString(output__, 3, *it_);
+ }
+ for (std::vector< std::string >::const_iterator it_ = outputs_.begin();
+ it_ != outputs_.end(); it_++) {
+ WriteString(output__, 4, *it_);
+ }
+ WriteString(output__, 5, desc_);
+ WriteString(output__, 6, command_);
+ WriteVarint32(output__, 7, console_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(id_) + 1;
+ size += VarintSize32(start_time_) + 1;
+ for (std::vector< std::string >::const_iterator it_ = inputs_.begin();
+ it_ != inputs_.end(); it_++) {
+ size += StringSize(*it_) + 1;
+ }
+ for (std::vector< std::string >::const_iterator it_ = outputs_.begin();
+ it_ != outputs_.end(); it_++) {
+ size += StringSize(*it_) + 1;
+ }
+ size += StringSize(desc_) + 1;
+ size += StringSize(command_) + 1;
+ size += VarintSizeBool(console_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ id_ = static_cast< uint32_t >(0);
+ start_time_ = static_cast< uint32_t >(0);
+ inputs_.clear();
+ outputs_.clear();
+ desc_.clear();
+ command_.clear();
+ console_ = static_cast< bool >(0);
+ }
+
+ uint32_t* mutable_id() {
+ has_id_ = true;
+ return &id_;
+ }
+ void set_id(const uint32_t& value) {
+ has_id_ = true;
+ id_ = value;
+ }
+ uint32_t* mutable_start_time() {
+ has_start_time_ = true;
+ return &start_time_;
+ }
+ void set_start_time(const uint32_t& value) {
+ has_start_time_ = true;
+ start_time_ = value;
+ }
+ std::vector< std::string >* mutable_inputs() {
+ has_inputs_ = true;
+ return &inputs_;
+ }
+ void add_inputs(const std::string& value) {
+ has_inputs_ = true;
+ inputs_.push_back(value);
+ }
+ void set_inputs(const std::vector< std::string >& value) {
+ has_inputs_ = true;
+ inputs_ = value;
+ }
+ std::vector< std::string >* mutable_outputs() {
+ has_outputs_ = true;
+ return &outputs_;
+ }
+ void add_outputs(const std::string& value) {
+ has_outputs_ = true;
+ outputs_.push_back(value);
+ }
+ void set_outputs(const std::vector< std::string >& value) {
+ has_outputs_ = true;
+ outputs_ = value;
+ }
+ std::string* mutable_desc() {
+ has_desc_ = true;
+ return &desc_;
+ }
+ void set_desc(const std::string& value) {
+ has_desc_ = true;
+ desc_ = value;
+ }
+ std::string* mutable_command() {
+ has_command_ = true;
+ return &command_;
+ }
+ void set_command(const std::string& value) {
+ has_command_ = true;
+ command_ = value;
+ }
+ bool* mutable_console() {
+ has_console_ = true;
+ return &console_;
+ }
+ void set_console(const bool& value) {
+ has_console_ = true;
+ console_ = value;
+ }
+ };
+
+ struct EdgeFinished {
+ uint32_t id_;
+ bool has_id_;
+ uint32_t end_time_;
+ bool has_end_time_;
+ int32_t status_;
+ bool has_status_;
+ std::string output_;
+ bool has_output_;
+ uint32_t user_time_;
+ bool has_user_time_;
+ uint32_t system_time_;
+ bool has_system_time_;
+
+ EdgeFinished() {
+ has_id_ = false;
+ id_ = static_cast< uint32_t >(0);
+ has_end_time_ = false;
+ end_time_ = static_cast< uint32_t >(0);
+ has_status_ = false;
+ status_ = static_cast< int32_t >(0);
+ has_output_ = false;
+ has_user_time_ = false;
+ user_time_ = static_cast< uint32_t >(0);
+ has_system_time_ = false;
+ system_time_ = static_cast< uint32_t >(0);
+ }
+
+ EdgeFinished(const EdgeFinished&);
+ void operator=(const EdgeFinished&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32(output__, 1, id_);
+ WriteVarint32(output__, 2, end_time_);
+ WriteVarint32(output__, 3, ZigZagEncode32(status_));
+ WriteString(output__, 4, output_);
+ WriteVarint32(output__, 5, user_time_);
+ WriteVarint32(output__, 6, system_time_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32(id_) + 1;
+ size += VarintSize32(end_time_) + 1;
+ size += VarintSize32(ZigZagEncode32(status_)) + 1;
+ size += StringSize(output_) + 1;
+ size += VarintSize32(user_time_) + 1;
+ size += VarintSize32(system_time_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ id_ = static_cast< uint32_t >(0);
+ end_time_ = static_cast< uint32_t >(0);
+ status_ = static_cast< int32_t >(0);
+ output_.clear();
+ user_time_ = static_cast< uint32_t >(0);
+ system_time_ = static_cast< uint32_t >(0);
+ }
+
+ uint32_t* mutable_id() {
+ has_id_ = true;
+ return &id_;
+ }
+ void set_id(const uint32_t& value) {
+ has_id_ = true;
+ id_ = value;
+ }
+ uint32_t* mutable_end_time() {
+ has_end_time_ = true;
+ return &end_time_;
+ }
+ void set_end_time(const uint32_t& value) {
+ has_end_time_ = true;
+ end_time_ = value;
+ }
+ int32_t* mutable_status() {
+ has_status_ = true;
+ return &status_;
+ }
+ void set_status(const int32_t& value) {
+ has_status_ = true;
+ status_ = value;
+ }
+ std::string* mutable_output() {
+ has_output_ = true;
+ return &output_;
+ }
+ void set_output(const std::string& value) {
+ has_output_ = true;
+ output_ = value;
+ }
+ uint32_t* mutable_user_time() {
+ has_user_time_ = true;
+ return &user_time_;
+ }
+ void set_user_time(const uint32_t& value) {
+ has_user_time_ = true;
+ user_time_ = value;
+ }
+ uint32_t* mutable_system_time() {
+ has_system_time_ = true;
+ return &system_time_;
+ }
+ void set_system_time(const uint32_t& value) {
+ has_system_time_ = true;
+ system_time_ = value;
+ }
+ };
+
+ struct Message {
+ enum Level {
+ INFO = 0,
+ WARNING = 1,
+ ERROR = 2,
+ DEBUG = 3,
+ };
+
+ ::ninja::Status::Message::Level level_;
+ bool has_level_;
+ std::string message_;
+ bool has_message_;
+
+ Message() {
+ has_level_ = false;
+ level_ = static_cast< ::ninja::Status::Message::Level >(0);
+ has_message_ = false;
+ }
+
+ Message(const Message&);
+ void operator=(const Message&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ WriteVarint32SignExtended(output__, 1, static_cast<int32_t>(level_));
+ WriteString(output__, 2, message_);
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ size += VarintSize32SignExtended(static_cast<int32_t>(level_)) + 1;
+ size += StringSize(message_) + 1;
+ return size;
+ }
+
+ void Clear() {
+ level_ = static_cast< ::ninja::Status::Message::Level >(0);
+ message_.clear();
+ }
+
+ ::ninja::Status::Message::Level* mutable_level() {
+ has_level_ = true;
+ return &level_;
+ }
+ void set_level(const ::ninja::Status::Message::Level& value) {
+ has_level_ = true;
+ level_ = value;
+ }
+ std::string* mutable_message() {
+ has_message_ = true;
+ return &message_;
+ }
+ void set_message(const std::string& value) {
+ has_message_ = true;
+ message_ = value;
+ }
+ };
+
+ ::ninja::Status::TotalEdges total_edges_;
+ bool has_total_edges_;
+ ::ninja::Status::BuildStarted build_started_;
+ bool has_build_started_;
+ ::ninja::Status::BuildFinished build_finished_;
+ bool has_build_finished_;
+ ::ninja::Status::EdgeStarted edge_started_;
+ bool has_edge_started_;
+ ::ninja::Status::EdgeFinished edge_finished_;
+ bool has_edge_finished_;
+ ::ninja::Status::Message message_;
+ bool has_message_;
+
+ Status() {
+ has_total_edges_ = false;
+ has_build_started_ = false;
+ has_build_finished_ = false;
+ has_edge_started_ = false;
+ has_edge_finished_ = false;
+ has_message_ = false;
+ }
+
+ Status(const Status&);
+ void operator=(const Status&);
+
+ void SerializeToOstream(std::ostream* output__) const {
+ if (has_total_edges_) {
+ WriteLengthDelimited(output__, 1,
+ total_edges_.ByteSizeLong());
+ total_edges_.SerializeToOstream(output__);
+ }
+ if (has_build_started_) {
+ WriteLengthDelimited(output__, 2,
+ build_started_.ByteSizeLong());
+ build_started_.SerializeToOstream(output__);
+ }
+ if (has_build_finished_) {
+ WriteLengthDelimited(output__, 3,
+ build_finished_.ByteSizeLong());
+ build_finished_.SerializeToOstream(output__);
+ }
+ if (has_edge_started_) {
+ WriteLengthDelimited(output__, 4,
+ edge_started_.ByteSizeLong());
+ edge_started_.SerializeToOstream(output__);
+ }
+ if (has_edge_finished_) {
+ WriteLengthDelimited(output__, 5,
+ edge_finished_.ByteSizeLong());
+ edge_finished_.SerializeToOstream(output__);
+ }
+ if (has_message_) {
+ WriteLengthDelimited(output__, 6,
+ message_.ByteSizeLong());
+ message_.SerializeToOstream(output__);
+ }
+ }
+
+ size_t ByteSizeLong() const {
+ size_t size = 0;
+ if (has_total_edges_) {
+ size += 1 + VarintSize32(total_edges_.ByteSizeLong());
+ size += total_edges_.ByteSizeLong();
+ }
+ if (has_build_started_) {
+ size += 1 + VarintSize32(build_started_.ByteSizeLong());
+ size += build_started_.ByteSizeLong();
+ }
+ if (has_build_finished_) {
+ size += 1 + VarintSize32(build_finished_.ByteSizeLong());
+ size += build_finished_.ByteSizeLong();
+ }
+ if (has_edge_started_) {
+ size += 1 + VarintSize32(edge_started_.ByteSizeLong());
+ size += edge_started_.ByteSizeLong();
+ }
+ if (has_edge_finished_) {
+ size += 1 + VarintSize32(edge_finished_.ByteSizeLong());
+ size += edge_finished_.ByteSizeLong();
+ }
+ if (has_message_) {
+ size += 1 + VarintSize32(message_.ByteSizeLong());
+ size += message_.ByteSizeLong();
+ }
+ return size;
+ }
+
+ void Clear() {
+ if (has_total_edges_) {
+ total_edges_.Clear();
+ has_total_edges_ = false;
+ }
+ if (has_build_started_) {
+ build_started_.Clear();
+ has_build_started_ = false;
+ }
+ if (has_build_finished_) {
+ build_finished_.Clear();
+ has_build_finished_ = false;
+ }
+ if (has_edge_started_) {
+ edge_started_.Clear();
+ has_edge_started_ = false;
+ }
+ if (has_edge_finished_) {
+ edge_finished_.Clear();
+ has_edge_finished_ = false;
+ }
+ if (has_message_) {
+ message_.Clear();
+ has_message_ = false;
+ }
+ }
+
+ ::ninja::Status::TotalEdges* mutable_total_edges() {
+ has_total_edges_ = true;
+ return &total_edges_;
+ }
+ ::ninja::Status::BuildStarted* mutable_build_started() {
+ has_build_started_ = true;
+ return &build_started_;
+ }
+ ::ninja::Status::BuildFinished* mutable_build_finished() {
+ has_build_finished_ = true;
+ return &build_finished_;
+ }
+ ::ninja::Status::EdgeStarted* mutable_edge_started() {
+ has_edge_started_ = true;
+ return &edge_started_;
+ }
+ ::ninja::Status::EdgeFinished* mutable_edge_finished() {
+ has_edge_finished_ = true;
+ return &edge_finished_;
+ }
+ ::ninja::Status::Message* mutable_message() {
+ has_message_ = true;
+ return &message_;
+ }
+};
+
+}
+#endif // NINJA_FRONTEND_PB_H
diff --git a/src/frontend.proto b/src/frontend.proto
new file mode 100644
index 0000000..57423d8
--- /dev/null
+++ b/src/frontend.proto
@@ -0,0 +1,88 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+option optimize_for = LITE_RUNTIME;
+
+package ninja;
+
+message Status {
+ message TotalEdges {
+ // New value for total edges in the build.
+ optional uint32 total_edges = 1;
+ }
+
+ message BuildStarted {
+ // Number of jobs Ninja will run in parallel.
+ optional uint32 parallelism = 1;
+ // Verbose value passed to ninja.
+ optional bool verbose = 2;
+ }
+
+ message BuildFinished {
+ }
+
+ message EdgeStarted {
+ // Edge identification number, unique to a Ninja run.
+ optional uint32 id = 1;
+ // Edge start time in milliseconds since Ninja started.
+ optional uint32 start_time = 2;
+ // List of edge inputs.
+ repeated string inputs = 3;
+ // List of edge outputs.
+ repeated string outputs = 4;
+ // Description field from the edge.
+ optional string desc = 5;
+ // Command field from the edge.
+ optional string command = 6;
+ // Edge uses console.
+ optional bool console = 7;
+ }
+
+ message EdgeFinished {
+ // Edge identification number, unique to a Ninja run.
+ optional uint32 id = 1;
+ // Edge end time in milliseconds since Ninja started.
+ optional uint32 end_time = 2;
+ // Exit status (0 for success).
+ optional sint32 status = 3;
+ // Edge output, may contain ANSI codes.
+ optional string output = 4;
+ // Number of milliseconds spent executing in user mode
+ optional uint32 user_time = 5;
+ // Number of milliseconds spent executing in kernel mode
+ optional uint32 system_time = 6;
+ }
+
+ message Message {
+ enum Level {
+ INFO = 0;
+ WARNING = 1;
+ ERROR = 2;
+ DEBUG = 3;
+ }
+ // Message priority level (DEBUG, INFO, WARNING, ERROR).
+ optional Level level = 1 [default = INFO];
+ // Info/warning/error message from Ninja.
+ optional string message = 2;
+ }
+
+ optional TotalEdges total_edges = 1;
+ optional BuildStarted build_started = 2;
+ optional BuildFinished build_finished = 3;
+ optional EdgeStarted edge_started = 4;
+ optional EdgeFinished edge_finished = 5;
+ optional Message message = 6;
+}
diff --git a/src/graph.cc b/src/graph.cc
index 9c2f784..6c773aa 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -14,6 +14,7 @@
#include "graph.h"
+#include <deque>
#include <assert.h>
#include <stdio.h>
@@ -22,22 +23,187 @@
#include "depfile_parser.h"
#include "deps_log.h"
#include "disk_interface.h"
-#include "manifest_parser.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
+bool Node::PrecomputeStat(DiskInterface* disk_interface, std::string* err) {
+ if (in_edge() != nullptr) {
+ if (in_edge()->IsPhonyOutput()) {
+ return true;
+ }
+ return (precomputed_mtime_ = disk_interface->LStat(path_.str(), nullptr, err)) != -1;
+ } else {
+ return (precomputed_mtime_ = disk_interface->Stat(path_.str(), err)) != -1;
+ }
+}
+
bool Node::Stat(DiskInterface* disk_interface, string* err) {
- return (mtime_ = disk_interface->Stat(path_, err)) != -1;
+ if (in_edge() != nullptr) {
+ assert(!in_edge()->IsPhonyOutput());
+ return (mtime_ = disk_interface->LStat(path_.str(), nullptr, err)) != -1;
+ } else {
+ return (mtime_ = disk_interface->Stat(path_.str(), err)) != -1;
+ }
}
-bool DependencyScan::RecomputeDirty(Node* node, string* err) {
- vector<Node*> stack;
- return RecomputeDirty(node, &stack, err);
+bool Node::LStat(DiskInterface* disk_interface, bool* is_dir, string* err) {
+ assert(in_edge() != nullptr);
+ assert(!in_edge()->IsPhonyOutput());
+ return (mtime_ = disk_interface->LStat(path_.str(), is_dir, err)) != -1;
}
-bool DependencyScan::RecomputeDirty(Node* node, vector<Node*>* stack,
- string* err) {
+bool DependencyScan::RecomputeNodesDirty(const std::vector<Node*>& initial_nodes,
+ std::vector<Node*>* validation_nodes,
+ std::string* err) {
+ METRIC_RECORD("dep scan");
+ std::vector<Node*> all_nodes;
+ std::vector<Edge*> all_edges;
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+
+ {
+ METRIC_RECORD("dep scan : collect nodes+edges");
+ for (Node* node : initial_nodes)
+ CollectPrecomputeLists(node, &all_nodes, &all_edges);
+ }
+
+ bool success = true;
+ if (!PrecomputeNodesDirty(all_nodes, all_edges, thread_pool.get(), err))
+ success = false;
+
+
+ std::deque<Node*> nodes(initial_nodes.begin(), initial_nodes.end());
+
+ if (success) {
+ METRIC_RECORD("dep scan : main pass");
+ std::vector<Node*> stack;
+ std::vector<Node*> new_validation_nodes;
+ while (!nodes.empty()) {
+ Node* node = nodes.front();
+ nodes.pop_front();
+
+ stack.clear();
+ new_validation_nodes.clear();
+ if (!RecomputeNodeDirty(node, &stack, &new_validation_nodes, err)) {
+ success = false;
+ break;
+ }
+ nodes.insert(nodes.end(), new_validation_nodes.begin(),
+ new_validation_nodes.end());
+ if (!new_validation_nodes.empty()) {
+ assert(validation_nodes &&
+ "validations require RecomputeDirty to be called with validation_nodes");
+ validation_nodes->insert(validation_nodes->end(),
+ new_validation_nodes.begin(),
+ new_validation_nodes.end());
+ }
+ }
+ }
+
+ {
+ // Ensure that the precomputed mtime information can't be used after this
+ // dependency scan finishes.
+ METRIC_RECORD("dep scan : clear pre-stat");
+ ParallelMap(thread_pool.get(), all_nodes, [](Node* node) {
+ node->ClearPrecomputedStat();
+ });
+ }
+
+ return success;
+}
+
+void DependencyScan::CollectPrecomputeLists(Node* node,
+ std::vector<Node*>* nodes,
+ std::vector<Edge*>* edges) {
+ if (node->precomputed_dirtiness())
+ return;
+ node->set_precomputed_dirtiness(true);
+ nodes->push_back(node);
+
+ Edge* edge = node->in_edge();
+ if (edge && !edge->precomputed_dirtiness_) {
+ edge->precomputed_dirtiness_ = true;
+ edges->push_back(edge);
+
+ for (Node* node : edge->inputs_) {
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call. (The precomputed_dirtiness() will be inlined, but the recursive
+ // call can't be.)
+ if (!node->precomputed_dirtiness())
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+
+ if (!edge->validations_.empty()) {
+ for (Node* node : edge->validations_) {
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call. (The precomputed_dirtiness() will be inlined, but the recursive
+ // call can't be.)
+ if (!node->precomputed_dirtiness())
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+ }
+ }
+
+ // Collect dependencies from the deps log. This pass could also examine
+ // depfiles, but it would be a more intrusive design change, because we don't
+ // want to parse a depfile twice.
+ if (DepsLog::Deps* deps = deps_log()->GetDeps(node)) {
+ for (int i = 0; i < deps->node_count; ++i) {
+ Node* node = deps->nodes[i];
+ // Duplicate the dirtiness check here to avoid an unnecessary function
+ // call.
+ if (!node->precomputed_dirtiness()) {
+ CollectPrecomputeLists(node, nodes, edges);
+ }
+ }
+ }
+}
+
+bool DependencyScan::PrecomputeNodesDirty(const std::vector<Node*>& nodes,
+ const std::vector<Edge*>& edges,
+ ThreadPool* thread_pool,
+ std::string* err) {
+ // Optimize the "null build" case by calling Stat in parallel on every node in
+ // the transitive closure.
+ //
+ // The Windows RealDiskInterface::Stat uses a directory-based cache that isn't
+ // thread-safe. Various tests also uses a non-thread-safe Stat, so disable the
+ // parallelized stat'ing for them as well.
+ if (disk_interface_->IsStatThreadSafe() &&
+ GetOptimalThreadPoolJobCount() > 1) {
+ METRIC_RECORD("dep scan : pre-stat nodes");
+ if (!PropagateError(err, ParallelMap(thread_pool, nodes,
+ [this](Node* node) {
+ // Each node is guaranteed to appear at most once in the collected list
+ // of nodes, so it's safe to modify the nodes from worker threads.
+ std::string err;
+ node->PrecomputeStat(disk_interface_, &err);
+ return err;
+ }))) {
+ return false;
+ }
+ }
+
+ {
+ METRIC_RECORD("dep scan : precompute edge info");
+ if (!PropagateError(err, ParallelMap(thread_pool, edges, [](Edge* edge) {
+ // As with the node list, each edge appears at most once in the
+ // collected list, so it's safe to modify the edges from worker threads.
+ std::string err;
+ edge->PrecomputeDepScanInfo(&err);
+ return err;
+ }))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool DependencyScan::RecomputeNodeDirty(Node* node, std::vector<Node*>* stack,
+ std::vector<Node*>* validation_nodes,
+ std::string* err) {
Edge* edge = node->in_edge();
if (!edge) {
// If we already visited this leaf node then we are done.
@@ -65,39 +231,66 @@
stack->push_back(node);
bool dirty = false;
+ bool phony_output = edge->IsPhonyOutput();
edge->outputs_ready_ = true;
edge->deps_missing_ = false;
- // Load output mtimes so we can compare them to the most recent input below.
- for (vector<Node*>::iterator o = edge->outputs_.begin();
- o != edge->outputs_.end(); ++o) {
- if (!(*o)->StatIfNecessary(disk_interface_, err))
+ if (phony_output) {
+ EXPLAIN("edge with output %s is a phony output, so is always dirty",
+ node->path().c_str());
+ dirty = true;
+
+ if (edge->UsesDepsLog() || edge->UsesDepfile()) {
+ *err = "phony output " + node->path() + " has deps, which does not make sense.";
return false;
+ }
+ } else {
+ // Load output mtimes so we can compare them to the most recent input below.
+ for (vector<Node*>::iterator o = edge->outputs_.begin();
+ o != edge->outputs_.end(); ++o) {
+ if (!(*o)->StatIfNecessary(disk_interface_, err))
+ return false;
+ }
+
+ if (!dep_loader_.LoadDeps(edge, err)) {
+ if (!err->empty())
+ return false;
+ // Failed to load dependency info: rebuild to regenerate it.
+ // LoadDeps() did EXPLAIN() already, no need to do it here.
+ dirty = edge->deps_missing_ = true;
+ }
}
- if (!dep_loader_.LoadDeps(edge, err)) {
- if (!err->empty())
- return false;
- // Failed to load dependency info: rebuild to regenerate it.
- // LoadDeps() did EXPLAIN() already, no need to do it here.
- dirty = edge->deps_missing_ = true;
- }
+ // Store any validation nodes from the edge for adding to the initial
+ // nodes. Don't recurse into them, that would trigger the dependency
+ // cycle detector if the validation node depends on this node.
+ // RecomputeNodesDirty will add the validation nodes to the initial nodes
+ // and recurse into them.
+ validation_nodes->insert(validation_nodes->end(),
+ edge->validations_.begin(), edge->validations_.end());
// Visit all inputs; we're dirty if any of the inputs are dirty.
Node* most_recent_input = NULL;
for (vector<Node*>::iterator i = edge->inputs_.begin();
i != edge->inputs_.end(); ++i) {
// Visit this input.
- if (!RecomputeDirty(*i, stack, err))
+ if (!RecomputeNodeDirty(*i, stack, validation_nodes, err))
return false;
// If an input is not ready, neither are our outputs.
- if (Edge* in_edge = (*i)->in_edge()) {
+ Edge* in_edge = (*i)->in_edge();
+ if (in_edge != nullptr) {
if (!in_edge->outputs_ready_)
edge->outputs_ready_ = false;
}
- if (!edge->is_order_only(i - edge->inputs_.begin())) {
+ if (!phony_output && !edge->is_order_only(i - edge->inputs_.begin())) {
+ if (in_edge != nullptr && in_edge->IsPhonyOutput()) {
+ *err = "real file '" + node->path() +
+ "' depends on phony output '" + (*i)->path() + "'\n";
+ return false;
+ }
+
// If a regular input is dirty (or missing), we're dirty.
// Otherwise consider mtime.
if ((*i)->dirty()) {
@@ -182,10 +375,30 @@
bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input,
bool* outputs_dirty, string* err) {
- string command = edge->EvaluateCommand(/*incl_rsp_file=*/true);
+ assert(!edge->IsPhonyOutput());
+
+ uint64_t command_hash = edge->GetCommandHash();
for (vector<Node*>::iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
- if (RecomputeOutputDirty(edge, most_recent_input, command, *o)) {
+ if (edge->is_phony()) {
+ // Phony edges don't write any output. Outputs are only dirty if
+ // there are no inputs and we're missing the output.
+ if (edge->inputs_.empty() && !(*o)->exists()) {
+ // For phony targets defined in the ninja file, error when using dirty phony edges.
+ // The phony edges automatically created from depfiles still need the old behavior.
+ if (missing_phony_is_err_ && !edge->phony_from_depfile_) {
+ *err = "output " + (*o)->path() + " of phony edge doesn't exist. Missing 'phony_output = true'?";
+ return false;
+ } else {
+ EXPLAIN("output %s of phony edge with no inputs doesn't exist",
+ (*o)->path().c_str());
+ *outputs_dirty = true;
+ return true;
+ }
+ }
+ continue;
+ }
+ if (RecomputeOutputDirty(edge, most_recent_input, command_hash, *o)) {
*outputs_dirty = true;
return true;
}
@@ -195,18 +408,9 @@
bool DependencyScan::RecomputeOutputDirty(Edge* edge,
Node* most_recent_input,
- const string& command,
+ uint64_t command_hash,
Node* output) {
- if (edge->is_phony()) {
- // Phony edges don't write any output. Outputs are only dirty if
- // there are no inputs and we're missing the output.
- if (edge->inputs_.empty() && !output->exists()) {
- EXPLAIN("output %s of phony edge with no inputs doesn't exist",
- output->path().c_str());
- return true;
- }
- return false;
- }
+ assert(!edge->is_phony());
BuildLog::LogEntry* entry = 0;
@@ -225,8 +429,8 @@
// build log. Use that mtime instead, so that the file will only be
// considered dirty if an input was modified since the previous run.
bool used_restat = false;
- if (edge->GetBindingBool("restat") && build_log() &&
- (entry = build_log()->LookupByOutput(output->path()))) {
+ if (edge->IsRestat() && build_log() &&
+ (entry = build_log()->LookupByOutput(output->path_hashed()))) {
output_mtime = entry->mtime;
used_restat = true;
}
@@ -242,10 +446,10 @@
}
if (build_log()) {
- bool generator = edge->GetBindingBool("generator");
- if (entry || (entry = build_log()->LookupByOutput(output->path()))) {
+ bool generator = edge->IsGenerator();
+ if (entry || (entry = build_log()->LookupByOutput(output->path_hashed()))) {
if (!generator &&
- BuildLog::LogEntry::HashCommand(command) != entry->command_hash) {
+ command_hash != entry->command_hash) {
// May also be dirty due to the command changing since the last build.
// But if this is a generator rule, the command changing does not make us
// dirty.
@@ -281,111 +485,180 @@
return true;
}
-/// An Env for an Edge, providing $in and $out.
-struct EdgeEnv : public Env {
- enum EscapeKind { kShellEscape, kDoNotEscape };
+static const HashedStrView kIn { "in" };
+static const HashedStrView kInNewline { "in_newline" };
+static const HashedStrView kOut { "out" };
- EdgeEnv(Edge* edge, EscapeKind escape)
- : edge_(edge), escape_in_out_(escape), recursive_(false) {}
- virtual string LookupVariable(const string& var);
-
- /// Given a span of Nodes, construct a list of paths suitable for a command
- /// line.
- string MakePathList(vector<Node*>::iterator begin,
- vector<Node*>::iterator end,
- char sep);
-
- private:
- vector<string> lookups_;
- Edge* edge_;
- EscapeKind escape_in_out_;
- bool recursive_;
-};
-
-string EdgeEnv::LookupVariable(const string& var) {
- if (var == "in" || var == "in_newline") {
+bool EdgeEval::EvaluateVariable(std::string* out_append,
+ const HashedStrView& var,
+ std::string* err) {
+ if (var == kIn || var == kInNewline) {
int explicit_deps_count = edge_->inputs_.size() - edge_->implicit_deps_ -
edge_->order_only_deps_;
- return MakePathList(edge_->inputs_.begin(),
- edge_->inputs_.begin() + explicit_deps_count,
- var == "in" ? ' ' : '\n');
- } else if (var == "out") {
+ AppendPathList(out_append,
+ edge_->inputs_.begin(),
+ edge_->inputs_.begin() + explicit_deps_count,
+ var == kIn ? ' ' : '\n');
+ return true;
+ } else if (var == kOut) {
int explicit_outs_count = edge_->outputs_.size() - edge_->implicit_outs_;
- return MakePathList(edge_->outputs_.begin(),
- edge_->outputs_.begin() + explicit_outs_count,
- ' ');
+ AppendPathList(out_append,
+ edge_->outputs_.begin(),
+ edge_->outputs_.begin() + explicit_outs_count,
+ ' ');
+ return true;
}
- if (recursive_) {
- vector<string>::const_iterator it;
- if ((it = find(lookups_.begin(), lookups_.end(), var)) != lookups_.end()) {
- string cycle;
- for (; it != lookups_.end(); ++it)
- cycle.append(*it + " -> ");
- cycle.append(var);
- Fatal(("cycle in rule variables: " + cycle).c_str());
+ if (edge_->EvaluateVariableSelfOnly(out_append, var))
+ return true;
+
+ // Search for a matching rule binding.
+ if (const std::string* binding_pattern = edge_->rule().GetBinding(var)) {
+ // Detect recursive rule variable usage.
+ if (recursion_count_ == kEvalRecursionLimit) {
+ std::string cycle = recursion_vars_[0].AsString();
+ for (int i = 1; i < kEvalRecursionLimit; ++i) {
+ cycle += " -> " + recursion_vars_[i].AsString();
+ if (recursion_vars_[i] == recursion_vars_[0])
+ break;
+ }
+ *err = "cycle in rule variables: " + cycle;
+ return false;
}
+ recursion_vars_[recursion_count_++] = var.str_view();
+
+ return EvaluateBindingOnRule(out_append, *binding_pattern, this, err);
}
- // See notes on BindingEnv::LookupWithFallback.
- const EvalString* eval = edge_->rule_->GetBinding(var);
- if (recursive_ && eval)
- lookups_.push_back(var);
-
- // In practice, variables defined on rules never use another rule variable.
- // For performance, only start checking for cycles after the first lookup.
- recursive_ = true;
- return edge_->env_->LookupWithFallback(var, eval, this);
+ // Fall back to the edge's enclosing scope.
+ if (eval_phase_ == EdgeEval::kParseTime) {
+ Scope::EvaluateVariableAtPos(out_append, var, edge_->pos_.scope_pos());
+ } else {
+ Scope::EvaluateVariable(out_append, var, edge_->pos_.scope());
+ }
+ return true;
}
-string EdgeEnv::MakePathList(vector<Node*>::iterator begin,
- vector<Node*>::iterator end,
- char sep) {
- string result;
- for (vector<Node*>::iterator i = begin; i != end; ++i) {
- if (!result.empty())
- result.push_back(sep);
- const string& path = (*i)->PathDecanonicalized();
+void EdgeEval::AppendPathList(std::string* out_append,
+ std::vector<Node*>::iterator begin,
+ std::vector<Node*>::iterator end,
+ char sep) {
+ for (auto it = begin; it != end; ++it) {
+ if (it != begin)
+ out_append->push_back(sep);
+
+ const string& path = (*it)->PathDecanonicalized();
if (escape_in_out_ == kShellEscape) {
#if _WIN32
- GetWin32EscapedString(path, &result);
+ GetWin32EscapedString(path, out_append);
#else
- GetShellEscapedString(path, &result);
+ GetShellEscapedString(path, out_append);
#endif
} else {
- result.append(path);
+ out_append->append(path);
}
}
- return result;
}
-string Edge::EvaluateCommand(bool incl_rsp_file) {
- string command = GetBinding("command");
+static const HashedStrView kCommand { "command" };
+static const HashedStrView kDepfile { "depfile" };
+static const HashedStrView kRspfile { "rspfile" };
+static const HashedStrView kRspFileContent { "rspfile_content" };
+
+bool Edge::EvaluateCommand(std::string* out_append, bool incl_rsp_file,
+ std::string* err) {
+ METRIC_RECORD("eval command");
+ if (!EvaluateVariable(out_append, kCommand, err))
+ return false;
if (incl_rsp_file) {
- string rspfile_content = GetBinding("rspfile_content");
- if (!rspfile_content.empty())
- command += ";rspfile=" + rspfile_content;
+ std::string rspfile_content;
+ if (!EvaluateVariable(&rspfile_content, kRspFileContent, err))
+ return false;
+ if (!rspfile_content.empty()) {
+ out_append->append(";rspfile=");
+ out_append->append(rspfile_content);
+ }
}
+ return true;
+}
+
+std::string Edge::EvaluateCommand(bool incl_rsp_file) {
+ std::string command;
+ std::string err;
+ if (!EvaluateCommand(&command, incl_rsp_file, &err))
+ Fatal("%s", err.c_str());
return command;
}
-string Edge::GetBinding(const string& key) {
- EdgeEnv env(this, EdgeEnv::kShellEscape);
- return env.LookupVariable(key);
+static const HashedStrView kRestat { "restat" };
+static const HashedStrView kGenerator { "generator" };
+static const HashedStrView kDeps { "deps" };
+static const HashedStrView kPhonyOutput { "phony_output" };
+
+bool Edge::PrecomputeDepScanInfo(std::string* err) {
+ if (dep_scan_info_.valid)
+ return true;
+
+ // Precompute boolean flags.
+ auto get_bool_var = [this, err](const HashedStrView& var,
+ EdgeEval::EscapeKind escape, bool* out) {
+ std::string value;
+ if (!EvaluateVariable(&value, var, err, EdgeEval::kFinalScope, escape))
+ return false;
+ *out = !value.empty();
+ return true;
+ };
+ if (!get_bool_var(kRestat, EdgeEval::kShellEscape, &dep_scan_info_.restat)) return false;
+ if (!get_bool_var(kGenerator, EdgeEval::kShellEscape, &dep_scan_info_.generator)) return false;
+ if (!get_bool_var(kDeps, EdgeEval::kShellEscape, &dep_scan_info_.deps)) return false;
+ if (!get_bool_var(kDepfile, EdgeEval::kDoNotEscape, &dep_scan_info_.depfile)) return false;
+ if (!get_bool_var(kPhonyOutput, EdgeEval::kShellEscape, &dep_scan_info_.phony_output)) return false;
+
+ // Precompute the command hash.
+ std::string command;
+ if (!EvaluateCommand(&command, /*incl_rsp_file=*/true, err))
+ return false;
+ dep_scan_info_.command_hash = BuildLog::LogEntry::HashCommand(command);
+
+ dep_scan_info_.valid = true;
+ return true;
}
-bool Edge::GetBindingBool(const string& key) {
- return !GetBinding(key).empty();
+/// Returns dependency-scanning info or exits with a fatal error.
+const Edge::DepScanInfo& Edge::ComputeDepScanInfo() {
+ std::string err;
+ if (!PrecomputeDepScanInfo(&err))
+ Fatal("%s", err.c_str());
+ return dep_scan_info_;
}
-string Edge::GetUnescapedDepfile() {
- EdgeEnv env(this, EdgeEnv::kDoNotEscape);
- return env.LookupVariable("depfile");
+bool Edge::EvaluateVariable(std::string* out_append, const HashedStrView& key,
+ std::string* err, EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape) {
+ EdgeEval eval(this, phase, escape);
+ return eval.EvaluateVariable(out_append, key, err);
}
-string Edge::GetUnescapedRspfile() {
- EdgeEnv env(this, EdgeEnv::kDoNotEscape);
- return env.LookupVariable("rspfile");
+std::string Edge::GetBindingImpl(const HashedStrView& key,
+ EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape) {
+ std::string result;
+ std::string err;
+ if (!EvaluateVariable(&result, key, &err, phase, escape))
+ Fatal("%s", err.c_str());
+ return result;
+}
+
+std::string Edge::GetBinding(const HashedStrView& key) {
+ return GetBindingImpl(key, EdgeEval::kFinalScope, EdgeEval::kShellEscape);
+}
+
+std::string Edge::GetUnescapedDepfile() {
+ return GetBindingImpl(kDepfile, EdgeEval::kFinalScope, EdgeEval::kDoNotEscape);
+}
+
+std::string Edge::GetUnescapedRspfile() {
+ return GetBindingImpl(kRspfile, EdgeEval::kFinalScope, EdgeEval::kDoNotEscape);
}
void Edge::Dump(const char* prefix) const {
@@ -399,6 +672,13 @@
i != outputs_.end() && *i != NULL; ++i) {
printf("%s ", (*i)->path().c_str());
}
+ if (!validations_.empty()) {
+ printf(" validations ");
+ for (vector<Node*>::const_iterator i = validations_.begin();
+ i != validations_.end() && *i != NULL; ++i) {
+ printf("%s ", (*i)->path().c_str());
+ }
+ }
if (pool_) {
if (!pool_->name().empty()) {
printf("(in pool '%s')", pool_->name().c_str());
@@ -422,7 +702,21 @@
// of the form "build a: phony ... a ...". Restrict our
// "phonycycle" diagnostic option to the form it used.
return is_phony() && outputs_.size() == 1 && implicit_outs_ == 0 &&
- implicit_deps_ == 0;
+ implicit_deps_ == 0 && order_only_deps_ == 0;
+}
+
+bool Edge::EvaluateVariableSelfOnly(std::string* out_append,
+ const HashedStrView& var) const {
+ // ninja allows declaring the same binding repeatedly on an edge. Use the
+ // last matching binding.
+ const auto it_end = unevaled_bindings_.rend();
+ for (auto it = unevaled_bindings_.rbegin(); it != it_end; ++it) {
+ if (var == it->first) {
+ EvaluateBindingInScope(out_append, it->second, pos_.scope_pos());
+ return true;
+ }
+ }
+ return false;
}
// static
@@ -440,6 +734,66 @@
return result;
}
+Node::~Node() {
+ EdgeList* node = out_edges_.load();
+ while (node != nullptr) {
+ EdgeList* next = node->next;
+ delete node;
+ node = next;
+ }
+}
+
+// Does the node have at least one out edge?
+bool Node::has_out_edge() const {
+ return out_edges_.load() != nullptr;
+}
+
+std::vector<Edge*> Node::GetOutEdges() const {
+ // Include out-edges from the manifest.
+ std::vector<Edge*> result;
+ for (EdgeList* node = out_edges_.load(); node != nullptr; node = node->next) {
+ result.push_back(node->edge);
+ }
+ std::sort(result.begin(), result.end(), EdgeCmp());
+
+ // Add extra out-edges from depfiles and the deps log. Preserve the order
+ // of these extra edges; don't sort them.
+ std::copy(dep_scan_out_edges_.begin(), dep_scan_out_edges_.end(),
+ std::back_inserter(result));
+
+ return result;
+}
+
+std::vector<Edge*> Node::GetValidationOutEdges() const {
+ std::vector<Edge*> result;
+ for (EdgeList* node = validation_out_edges_.load(); node != nullptr; node = node->next) {
+ result.push_back(node->edge);
+ }
+ std::sort(result.begin(), result.end(), EdgeCmp());
+
+ return result;
+}
+
+void Node::AddOutEdge(Edge* edge) {
+ EdgeList* new_node = new EdgeList { edge };
+ while (true) {
+ EdgeList* cur_head = out_edges_.load();
+ new_node->next = cur_head;
+ if (out_edges_.compare_exchange_weak(cur_head, new_node))
+ break;
+ }
+}
+
+void Node::AddValidationOutEdge(Edge* edge) {
+ EdgeList* new_node = new EdgeList { edge };
+ while (true) {
+ EdgeList* cur_head = validation_out_edges_.load();
+ new_node->next = cur_head;
+ if (validation_out_edges_.compare_exchange_weak(cur_head, new_node))
+ break;
+ }
+}
+
void Node::Dump(const char* prefix) const {
printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ",
prefix, path().c_str(), this,
@@ -451,20 +805,31 @@
printf("no in-edge\n");
}
printf(" out edges:\n");
- for (vector<Edge*>::const_iterator e = out_edges().begin();
- e != out_edges().end() && *e != NULL; ++e) {
+ const std::vector<Edge*> out_edges = GetOutEdges();
+ for (vector<Edge*>::const_iterator e = out_edges.begin();
+ e != out_edges.end() && *e != NULL; ++e) {
(*e)->Dump(" +- ");
}
+ const std::vector<Edge*> validation_out_edges = GetValidationOutEdges();
+ if (!validation_out_edges.empty()) {
+ printf(" validation out edges:\n");
+ for (vector<Edge*>::const_iterator e = validation_out_edges.begin();
+ e != validation_out_edges.end() && *e != NULL; ++e) {
+ (*e)->Dump(" +- ");
+ }
+ }
}
bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) {
- string deps_type = edge->GetBinding("deps");
- if (!deps_type.empty())
+ if (edge->UsesDepsLog())
return LoadDepsFromLog(edge, err);
- string depfile = edge->GetUnescapedDepfile();
- if (!depfile.empty())
+ if (edge->UsesDepfile()) {
+ std::string depfile = edge->GetUnescapedDepfile();
+ assert(!depfile.empty() &&
+ "UsesDepfile was set, so the depfile should be non-empty");
return LoadDepFile(edge, depfile, err);
+ }
// No deps to load.
return true;
@@ -510,8 +875,7 @@
// Check that this depfile matches the edge's output, if not return false to
// mark the edge as dirty.
Node* first_output = edge->outputs_[0];
- StringPiece opath = StringPiece(first_output->path());
- if (opath != depfile.out_) {
+ if (first_output->path() != depfile.out_) {
EXPLAIN("expected depfile '%s' to mention '%s', got '%s'", path.c_str(),
first_output->path().c_str(), depfile.out_.AsString().c_str());
return false;
@@ -531,7 +895,7 @@
Node* node = state_->GetNode(*i, slash_bits);
*implicit_dep = node;
- node->AddOutEdge(edge);
+ node->AddOutEdgeDepScan(edge);
CreatePhonyInEdge(node);
}
@@ -559,7 +923,7 @@
for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) {
Node* node = deps->nodes[i];
*implicit_dep = node;
- node->AddOutEdge(edge);
+ node->AddOutEdgeDepScan(edge);
CreatePhonyInEdge(node);
}
return true;
@@ -580,6 +944,7 @@
Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
node->set_in_edge(phony_edge);
phony_edge->outputs_.push_back(node);
+ ++phony_edge->explicit_outs_;
// RecomputeDirty might not be called for phony_edge if a previous call
// to RecomputeDirty had caused the file to be stat'ed. Because previous
@@ -588,4 +953,6 @@
// to avoid a potential stuck build. If we do call RecomputeDirty for
// this node, it will simply set outputs_ready_ to the correct value.
phony_edge->outputs_ready_ = true;
+
+ phony_edge->phony_from_depfile_ = true;
}
diff --git a/src/graph.h b/src/graph.h
index d58fecd..b648947 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -15,6 +15,8 @@
#ifndef NINJA_GRAPH_H_
#define NINJA_GRAPH_H_
+#include <atomic>
+#include <set>
#include <string>
#include <vector>
using namespace std;
@@ -31,32 +33,108 @@
struct Node;
struct Pool;
struct State;
+struct ThreadPool;
+
+/// A reference to a path from the lexer. This path is unevaluated, stored in
+/// mmap'ed memory, and is guaranteed to be followed by a terminating character
+/// (e.g. whitespace, a colon or pipe, etc).
+struct LexedPath {
+ StringPiece str_;
+};
+
+#ifdef _WIN32
+
+/// On Windows, we record the '/'-vs-'\' slash direction in the first reference
+/// to a path, which determines the path strings used in command-lines.
+struct NodeSlashBits {
+ NodeSlashBits() {}
+ NodeSlashBits(uint64_t value) : slash_bits_(value) {}
+ uint64_t slash_bits() const { return slash_bits_; }
+private:
+ uint64_t slash_bits_ = 0;
+};
+
+#else
+
+/// By default, '\' is not recognized as a path separator, and slash_bits is
+/// always 0.
+struct NodeSlashBits {
+ NodeSlashBits() {}
+ NodeSlashBits(uint64_t /*value*/) {}
+ uint64_t slash_bits() const { return 0; }
+};
+
+#endif // _WIN32
+
+/// Record the earliest reference to the node, which is needed for two uses:
+///
+/// - To verify that a node exists before a "default" declaration references
+/// it.
+/// - On Windows, the earliest reference to the node determines the slash_bits
+/// value for decanonicalizing the node's path.
+///
+/// On Windows, atomic operations on this struct probably use a spin lock, but
+/// it can be configured to use a DWCAS (e.g. -mcx16 with Clang/libc++). On
+/// other targets, the empty base class occupies 0 bytes, and atomic operations
+/// will be lock-free.
+struct NodeFirstReference : NodeSlashBits {
+ NodeFirstReference() {}
+ NodeFirstReference(DeclIndex loc, uint64_t slash_bits)
+ : NodeSlashBits(slash_bits), loc_(loc) {}
+ DeclIndex dfs_location() const { return loc_; }
+private:
+ DeclIndex loc_ = kLastDeclIndex;
+};
+
+inline bool operator<(const NodeFirstReference& x,
+ const NodeFirstReference& y) {
+ if (x.dfs_location() < y.dfs_location()) return true;
+ if (x.dfs_location() > y.dfs_location()) return false;
+ return x.slash_bits() < y.slash_bits();
+}
/// Information about a node in the dependency graph: the file, whether
/// it's dirty, mtime, etc.
struct Node {
- Node(const string& path, uint64_t slash_bits)
+ Node(const HashedStrView& path, uint64_t initial_slash_bits)
: path_(path),
- slash_bits_(slash_bits),
- mtime_(-1),
- dirty_(false),
- in_edge_(NULL),
- id_(-1) {}
+ first_reference_({ kLastDeclIndex, initial_slash_bits }) {}
+ ~Node();
+
+ /// Precompute the node's Stat() call from a worker thread with exclusive
+ /// access to this node. Returns false on error.
+ bool PrecomputeStat(DiskInterface* disk_interface, string* err);
+
+ /// After the dependency scan is complete, reset the precomputed mtime so it
+ /// can't affect later StatIfNecessary() calls.
+ void ClearPrecomputedStat() {
+ precomputed_mtime_ = -1;
+ }
/// Return false on error.
+ /// Uses stat() or lstat() as appropriate.
bool Stat(DiskInterface* disk_interface, string* err);
+ /// Only use when lstat() is desired (output files)
+ bool LStat(DiskInterface* disk_interface, bool* is_dir, string* err);
+
/// Return false on error.
bool StatIfNecessary(DiskInterface* disk_interface, string* err) {
if (status_known())
return true;
+ if (precomputed_mtime_ >= 0) {
+ mtime_ = precomputed_mtime_;
+ return true;
+ }
return Stat(disk_interface, err);
}
/// Mark as not-yet-stat()ed and not dirty.
void ResetState() {
mtime_ = -1;
+ precomputed_mtime_ = -1;
dirty_ = false;
+ precomputed_dirtiness_ = false;
}
/// Mark the Node as already-stat()ed and missing.
@@ -72,14 +150,14 @@
return mtime_ != -1;
}
- const string& path() const { return path_; }
+ const std::string& path() const { return path_.str(); }
+ const HashedStr& path_hashed() const { return path_; }
/// Get |path()| but use slash_bits to convert back to original slash styles.
string PathDecanonicalized() const {
- return PathDecanonicalized(path_, slash_bits_);
+ return PathDecanonicalized(path_.str(), slash_bits());
}
static string PathDecanonicalized(const string& path,
uint64_t slash_bits);
- uint64_t slash_bits() const { return slash_bits_; }
TimeStamp mtime() const { return mtime_; }
@@ -87,44 +165,131 @@
void set_dirty(bool dirty) { dirty_ = dirty; }
void MarkDirty() { dirty_ = true; }
+ bool precomputed_dirtiness() const { return precomputed_dirtiness_; }
+ void set_precomputed_dirtiness(bool value) { precomputed_dirtiness_ = value; }
+
Edge* in_edge() const { return in_edge_; }
void set_in_edge(Edge* edge) { in_edge_ = edge; }
int id() const { return id_; }
void set_id(int id) { id_ = id; }
- const vector<Edge*>& out_edges() const { return out_edges_; }
- void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); }
+ // Thread-safe properties.
+ DeclIndex dfs_location() const {
+ return first_reference_.load().dfs_location();
+ }
+ uint64_t slash_bits() const {
+ return first_reference_.load().slash_bits();
+ }
+ void UpdateFirstReference(DeclIndex dfs_location, uint64_t slash_bits) {
+ AtomicUpdateMinimum(&first_reference_, { dfs_location, slash_bits });
+ }
+
+ // Thread-safe properties.
+ bool has_out_edge() const;
+ std::vector<Edge*> GetOutEdges() const;
+ std::vector<Edge*> GetValidationOutEdges() const;
+ void AddOutEdge(Edge* edge);
+ void AddValidationOutEdge(Edge* edge);
+
+ /// Add an out-edge from the dependency scan. This function differs from
+ /// AddOutEdge in several ways:
+ /// - It uses a simple vector, which is faster for single-threaded use.
+ /// - It's not thread-safe.
+ /// - It preserves edge order. Edges added with AddOutEdge come from the
+ /// manifest and are ordered by their position within the manifest
+ /// (represented with the edge ID). Dep scan edges, on the other hand,
+ /// are ordered by a DFS walk from target nodes to their dependencies.
+ /// (I'm not sure whether this order is practically important.)
+ void AddOutEdgeDepScan(Edge* edge) { dep_scan_out_edges_.push_back(edge); }
void Dump(const char* prefix="") const;
-private:
- string path_;
+ // Used in the inputs debug tool.
+ bool InputsChecked() const { return inputs_checked_; }
+ void MarkInputsChecked() { inputs_checked_ = true; }
- /// Set bits starting from lowest for backslashes that were normalized to
- /// forward slashes by CanonicalizePath. See |PathDecanonicalized|.
- uint64_t slash_bits_;
+private:
+ const HashedStr path_;
/// Possible values of mtime_:
/// -1: file hasn't been examined
/// 0: we looked, and file doesn't exist
/// >0: actual file's mtime
- TimeStamp mtime_;
+ TimeStamp mtime_ = -1;
+
+ /// If this value is >= 0, it represents a precomputed mtime for the node.
+ TimeStamp precomputed_mtime_ = -1;
/// Dirty is true when the underlying file is out-of-date.
/// But note that Edge::outputs_ready_ is also used in judging which
/// edges to build.
- bool dirty_;
+ bool dirty_ = false;
- /// The Edge that produces this Node, or NULL when there is no
- /// known edge to produce it.
- Edge* in_edge_;
-
- /// All Edges that use this Node as an input.
- vector<Edge*> out_edges_;
+ /// Set to true once the node's stat and command-hash info have been
+ /// precomputed.
+ bool precomputed_dirtiness_ = false;
/// A dense integer id for the node, assigned and used by DepsLog.
- int id_;
+ int id_ = -1;
+
+ std::atomic<NodeFirstReference> first_reference_;
+
+ Edge* in_edge_ = nullptr;
+
+ struct EdgeList {
+ EdgeList(Edge* edge=nullptr, EdgeList* next=nullptr)
+ : edge(edge), next(next) {}
+
+ Edge* edge = nullptr;
+ EdgeList* next = nullptr;
+ };
+
+ /// All Edges that use this Node as an input. The order of this list is
+ /// non-deterministic. An accessor function sorts it each time it's used.
+ std::atomic<EdgeList*> out_edges_ { nullptr };
+ std::atomic<EdgeList*> validation_out_edges_ { nullptr };
+
+ std::vector<Edge*> dep_scan_out_edges_;
+
+ /// Stores if this node's inputs have been already computed. Used in the
+ /// inputs debug tool.
+ bool inputs_checked_ = false;
+};
+
+struct EdgeEval {
+ enum EvalPhase { kParseTime, kFinalScope };
+ enum EscapeKind { kShellEscape, kDoNotEscape };
+
+ EdgeEval(Edge* edge, EvalPhase eval_phase, EscapeKind escape)
+ : edge_(edge),
+ eval_phase_(eval_phase),
+ escape_in_out_(escape) {}
+
+ /// Looks up the variable and appends its value to the output buffer. Returns
+ /// false on error (i.e. a cycle in rule variable expansion).
+ bool EvaluateVariable(std::string* out_append, const HashedStrView& var,
+ std::string* err);
+
+ /// There are only a small number of bindings allowed on a rule. If we recurse
+ /// enough times, we're guaranteed to repeat a variable.
+ static constexpr int kEvalRecursionLimit = 16;
+
+private:
+ Edge* edge_ = nullptr;
+
+ EvalPhase eval_phase_ = kFinalScope;
+
+ /// The kind of escaping to do on $in and $out path variables.
+ EscapeKind escape_in_out_ = kShellEscape;
+
+ int recursion_count_ = 0;
+ StringPiece recursion_vars_[kEvalRecursionLimit];
+
+ void AppendPathList(std::string* out_append,
+ std::vector<Node*>::iterator begin,
+ std::vector<Node*>::iterator end,
+ char sep);
};
/// An edge in the dependency graph; links between Nodes using Rules.
@@ -135,9 +300,15 @@
VisitDone
};
- Edge() : rule_(NULL), pool_(NULL), env_(NULL), mark_(VisitNone),
- outputs_ready_(false), deps_missing_(false),
- implicit_deps_(0), order_only_deps_(0), implicit_outs_(0) {}
+ struct DepScanInfo {
+ bool valid = false;
+ bool restat = false;
+ bool generator = false;
+ bool deps = false;
+ bool depfile = false;
+ bool phony_output = false;
+ uint64_t command_hash = 0;
+ };
/// Return true if all inputs' in-edges are ready.
bool AllInputsReady() const;
@@ -145,12 +316,45 @@
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
- string EvaluateCommand(bool incl_rsp_file = false);
+ bool EvaluateCommand(std::string* out_append, bool incl_rsp_file,
+ std::string* err);
- /// Returns the shell-escaped value of |key|.
- string GetBinding(const string& key);
- bool GetBindingBool(const string& key);
+ /// Convenience method. This method must not be called from a worker thread,
+ /// because it could abort with a fatal error. (For consistency with other
+ /// Get*/Evaluate* methods, a better name might be GetCommand.)
+ std::string EvaluateCommand(bool incl_rsp_file = false);
+ /// Attempts to evaluate info needed for scanning dependencies.
+ bool PrecomputeDepScanInfo(std::string* err);
+
+ /// Returns dependency-scanning info or exits with a fatal error. These
+ /// methods must not be called until after the manifest has been loaded.
+ const DepScanInfo& ComputeDepScanInfo();
+ uint64_t GetCommandHash() { return ComputeDepScanInfo().command_hash; }
+ bool IsRestat() { return ComputeDepScanInfo().restat; }
+ bool IsGenerator() { return ComputeDepScanInfo().generator; }
+ bool IsPhonyOutput() { return ComputeDepScanInfo().phony_output; }
+ bool UsesDepsLog() { return ComputeDepScanInfo().deps; }
+ bool UsesDepfile() { return ComputeDepScanInfo().depfile; }
+
+ /// Appends the value of |key| to the output buffer. On error, returns false,
+ /// and the content of the output buffer is unspecified.
+ bool EvaluateVariable(std::string* out_append, const HashedStrView& key,
+ std::string* err,
+ EdgeEval::EvalPhase phase=EdgeEval::kFinalScope,
+ EdgeEval::EscapeKind escape=EdgeEval::kShellEscape);
+
+private:
+ std::string GetBindingImpl(const HashedStrView& key,
+ EdgeEval::EvalPhase phase,
+ EdgeEval::EscapeKind escape);
+
+public:
+ /// Convenience method for EvaluateVariable. On failure, it issues a fatal
+ /// error. This function must not be called from a worker thread because:
+ /// - Error reporting should be deterministic, and
+ /// - Fatal() destructs static globals, which a worker thread could be using.
+ std::string GetBinding(const HashedStrView& key);
/// Like GetBinding("depfile"), but without shell escaping.
string GetUnescapedDepfile();
/// Like GetBinding("rspfile"), but without shell escaping.
@@ -158,15 +362,45 @@
void Dump(const char* prefix="") const;
- const Rule* rule_;
- Pool* pool_;
+ /// Temporary fields used only during manifest parsing.
+ struct DeferredPathList {
+ enum Type {
+ INPUT = 0,
+ OUTPUT = 1,
+ VALIDATION = 2,
+ };
+
+ DeferredPathList(const char* lexer_pos=nullptr,
+ Type type = INPUT, int count=0)
+ : lexer_pos(lexer_pos), type(type), count(count) {}
+
+ const char* lexer_pos = nullptr;
+ Type type;
+ int count = 0;
+ };
+ struct {
+ StringPiece rule_name;
+ size_t rule_name_diag_pos = 0;
+ size_t final_diag_pos = 0;
+ std::vector<DeferredPathList> deferred_path_lists;
+ } parse_state_;
+
+ RelativePosition pos_;
+ const Rule* rule_ = nullptr;
+ Pool* pool_ = nullptr;
vector<Node*> inputs_;
vector<Node*> outputs_;
- BindingEnv* env_;
- VisitMark mark_;
- bool outputs_ready_;
- bool deps_missing_;
+ vector<Node*> validations_;
+ std::vector<std::pair<HashedStr, std::string>> unevaled_bindings_;
+ VisitMark mark_ = VisitNone;
+ bool precomputed_dirtiness_ = false;
+ size_t id_ = 0;
+ bool outputs_ready_ = false;
+ bool deps_missing_ = false;
+ bool phony_from_depfile_ = false;
+ DepScanInfo dep_scan_info_;
+ DeclIndex dfs_location() const { return pos_.dfs_location(); }
const Rule& rule() const { return *rule_; }
Pool* pool() const { return pool_; }
int weight() const { return 1; }
@@ -180,8 +414,9 @@
// don't cause the target to rebuild.
// These are stored in inputs_ in that order, and we keep counts of
// #2 and #3 when we need to access the various subsets.
- int implicit_deps_;
- int order_only_deps_;
+ int explicit_deps_ = 0;
+ int implicit_deps_ = 0;
+ int order_only_deps_ = 0;
bool is_implicit(size_t index) {
return index >= inputs_.size() - order_only_deps_ - implicit_deps_ &&
!is_order_only(index);
@@ -195,16 +430,32 @@
// 2) implicit outs, which the target generates but are not part of $out.
// These are stored in outputs_ in that order, and we keep a count of
// #2 to use when we need to access the various subsets.
- int implicit_outs_;
+ int explicit_outs_ = 0;
+ int implicit_outs_ = 0;
bool is_implicit_out(size_t index) const {
return index >= outputs_.size() - implicit_outs_;
}
+ int validation_deps_ = 0;
+
bool is_phony() const;
bool use_console() const;
bool maybe_phonycycle_diagnostic() const;
+
+ /// Search for a binding on this edge and append its value to the output
+ /// string. Does not search the enclosing scope. Use other functions to
+ /// include ancestor scopes and rule bindings.
+ bool EvaluateVariableSelfOnly(std::string* out_append,
+ const HashedStrView& var) const;
};
+struct EdgeCmp {
+ bool operator()(const Edge* a, const Edge* b) const {
+ return a->id_ < b->id_;
+ }
+};
+
+typedef set<Edge*, EdgeCmp> EdgeSet;
/// ImplicitDepLoader loads implicit dependencies, as referenced via the
/// "depfile" attribute in build files.
@@ -254,17 +505,30 @@
struct DependencyScan {
DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log,
DiskInterface* disk_interface,
- DepfileParserOptions const* depfile_parser_options)
+ DepfileParserOptions const* depfile_parser_options,
+ bool missing_phony_is_err)
: build_log_(build_log),
disk_interface_(disk_interface),
- dep_loader_(state, deps_log, disk_interface, depfile_parser_options) {}
+ dep_loader_(state, deps_log, disk_interface, depfile_parser_options),
+ missing_phony_is_err_(missing_phony_is_err) {}
- /// Update the |dirty_| state of the given node by inspecting its input edge.
+ /// Used for tests.
+ bool RecomputeDirty(Node* node, std::vector<Node*>* validation_nodes,
+ std::string* err) {
+ std::vector<Node*> nodes = {node};
+ return RecomputeNodesDirty(nodes, validation_nodes, err);
+ }
+
+ /// Update the |dirty_| state of the given nodes by transitively inspecting
+ /// their input edges.
/// Examine inputs, outputs, and command lines to judge whether an edge
/// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_|
/// state accordingly.
+ /// Appends any validation nodes found to the nodes parameter.
/// Returns false on failure.
- bool RecomputeDirty(Node* node, string* err);
+ bool RecomputeNodesDirty(const std::vector<Node*>& initial_nodes,
+ std::vector<Node*>* validation_nodes,
+ std::string* err);
/// Recompute whether any output of the edge is dirty, if so sets |*dirty|.
/// Returns false on failure.
@@ -283,17 +547,32 @@
}
private:
- bool RecomputeDirty(Node* node, vector<Node*>* stack, string* err);
+ /// Find the transitive closure of edges and nodes that the given node depends
+ /// on. Each Node and Edge is guaranteed to appear at most once in an output
+ /// vector. The returned lists are not guaranteed to be a superset or a subset
+ /// of the nodes and edges that RecomputeNodesDirty will initialize.
+ void CollectPrecomputeLists(Node* node, std::vector<Node*>* nodes,
+ std::vector<Edge*>* edges);
+
+ bool PrecomputeNodesDirty(const std::vector<Node*>& nodes,
+ const std::vector<Edge*>& edges,
+ ThreadPool* thread_pool, std::string* err);
+
+ bool RecomputeNodeDirty(Node* node, vector<Node*>* stack,
+ vector<Node*>* validation_nodes, string* err);
+
bool VerifyDAG(Node* node, vector<Node*>* stack, string* err);
/// Recompute whether a given single output should be marked dirty.
/// Returns true if so.
bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input,
- const string& command, Node* output);
+ uint64_t command_hash, Node* output);
BuildLog* build_log_;
DiskInterface* disk_interface_;
ImplicitDepLoader dep_loader_;
+
+ bool missing_phony_is_err_;
};
#endif // NINJA_GRAPH_H_
diff --git a/src/graph_test.cc b/src/graph_test.cc
index 4a66831..a957513 100644
--- a/src/graph_test.cc
+++ b/src/graph_test.cc
@@ -18,7 +18,7 @@
#include "test.h"
struct GraphTest : public StateTestWithBuiltinRules {
- GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL) {}
+ GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL, false) {}
VirtualFileSystem fs_;
DependencyScan scan_;
@@ -31,7 +31,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
// A missing implicit dep *should* make the output dirty.
@@ -49,7 +49,7 @@
fs_.Create("implicit", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
// A modified implicit dep should make the output dirty.
@@ -69,7 +69,7 @@
fs_.Create("implicit.h", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
// implicit.h has changed, though our depfile refers to it with a
@@ -92,7 +92,7 @@
fs_.Create("data", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
// We have both an implicit and an explicit dep on implicit.h.
@@ -120,7 +120,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out")->dirty());
@@ -136,7 +136,7 @@
fs_.Create("out", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out")->dirty());
@@ -160,7 +160,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.imp")->dirty());
@@ -174,7 +174,7 @@
fs_.Create("in", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.imp")->dirty());
@@ -191,7 +191,7 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
@@ -239,7 +239,7 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
@@ -259,13 +259,13 @@
fs_.Create("out.o", "");
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_FALSE(GetNode("out.o")->dirty());
state_.Reset();
fs_.RemoveFile("out.o.d");
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err));
ASSERT_EQ("", err);
EXPECT_TRUE(GetNode("out.o")->dirty());
}
@@ -312,7 +312,7 @@
"build n2: phony n1\n"
);
string err;
- EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), &err));
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), NULL, &err));
ASSERT_EQ("", err);
Plan plan_;
@@ -331,10 +331,190 @@
parser_opts);
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err);
}
+TEST_F(GraphTest, OutputSymlinkSourceUpdate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build other: cat\n"
+"build sym: cat\n"
+"build out: cat | sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "other");
+ fs_.Tick();
+ fs_.Create("other", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, OutputSymlinkDangling) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build sym: cat\n"
+"build out: cat | sym\n"));
+
+ fs_.CreateSymlink("sym", "dangling");
+ fs_.Create("out", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlink) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "in");
+ fs_.Tick();
+ fs_.Create("in", "");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlinkUpdate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.Create("in", "");
+ fs_.Tick();
+ fs_.CreateSymlink("sym", "in");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ // This can be incorrect if the destination of the symlink changed to
+ // a file with an equal or older timestamp.
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputSymlinkDangling) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat sym\n"));
+
+ fs_.Create("out", "");
+ fs_.CreateSymlink("sym", "in");
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputDirectoryUpToDate) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat inputs\n"));
+
+ fs_.Create("out", "");
+ EXPECT_TRUE(fs_.MakeDir("inputs"));
+ EXPECT_TRUE(fs_.WriteFile("inputs/foo", ""));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_FALSE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, InputDirectoryChanged) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat inputs\n"));
+
+ fs_.Create("out", "");
+ EXPECT_TRUE(fs_.MakeDir("inputs"));
+ fs_.Tick();
+ EXPECT_TRUE(fs_.WriteFile("inputs/foo", ""));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+}
+
+TEST_F(GraphTest, PhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"));
+
+ Node* node = state_.LookupNode("foo");
+ Edge* edge = node->in_edge();
+ ASSERT_TRUE(edge->IsPhonyOutput());
+}
+
+TEST_F(GraphTest, PhonyOutputDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"
+"build bar: phony_out foo\n"));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ ASSERT_EQ("", err);
+}
+
+TEST_F(GraphTest, RealDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"rule touch\n"
+" command = touch ${out}\n"
+"build foo: phony_out\n"
+"build bar: touch foo\n"));
+
+ string err;
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ EXPECT_EQ("real file 'bar' depends on phony output 'foo'\n", err);
+}
+
+TEST_F(GraphTest, PhonyDependsOnPhonyOutput) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"
+"build bar: phony foo\n"));
+
+ string err;
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("bar"), NULL, &err));
+ EXPECT_EQ("real file 'bar' depends on phony output 'foo'\n", err);
+}
+
+TEST_F(GraphTest, MissingPhonyWithPhonyOutputs) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build foo: phony\n"));
+
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("foo"), NULL, &err));
+ EXPECT_EQ("", err);
+ EXPECT_TRUE(GetNode("foo")->dirty());
+
+ state_.Reset();
+ DependencyScan scan(&state_, NULL, NULL, &fs_, NULL, true);
+ EXPECT_FALSE(scan.RecomputeDirty(GetNode("foo"), NULL, &err));
+ EXPECT_EQ("output foo of phony edge doesn't exist. Missing 'phony_output = true'?", err);
+}
+
TEST_F(GraphTest, DependencyCycle) {
AssertParse(&state_,
"build out: cat mid\n"
@@ -343,7 +523,7 @@
"build pre: cat out\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err));
ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err);
}
@@ -351,7 +531,7 @@
string err;
AssertParse(&state_,
"build a b: cat a\n");
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a", err);
}
@@ -359,7 +539,7 @@
string err;
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build b a: cat a\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> a", err);
}
@@ -368,7 +548,7 @@
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
"build a b: cat c\n"
"build c: cat a\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> c -> a", err);
}
@@ -380,7 +560,7 @@
"build b: cat a\n"
"build a e: cat d\n"
"build f: cat e\n"));
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), NULL, &err));
ASSERT_EQ("dependency cycle: a -> d -> c -> b -> a", err);
}
@@ -396,7 +576,7 @@
fs_.Create("dep.d", "a: b\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> b", err);
// Despite the depfile causing edge to be a cycle (it has outputs a and b,
@@ -421,7 +601,7 @@
fs_.Create("dep.d", "a: c\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> c -> b", err);
// Despite the depfile causing edge to be a cycle (|edge| has outputs a and b,
@@ -448,7 +628,7 @@
fs_.Create("dep.d", "a: c\n");
string err;
- EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), &err));
+ EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), NULL, &err));
ASSERT_EQ("dependency cycle: b -> c -> b", err);
// Despite the depfile causing edge to be a cycle (|edge| has outputs a and b,
@@ -479,3 +659,68 @@
EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo");
}
#endif
+
+TEST_F(GraphTest, EdgeVarEvalPhase) {
+ // Variable lookups on edges can happen in one of two phases:
+ // - Parse-time: Only bindings declared before the reference are visible.
+ // - Final-scope: All bindings in the current scope (and ancestor scopes)
+ // are visible.
+ //
+ // An edge's pool is determined at parse-time, while most other bindings are
+ // looked up after manifest parsing is finished.
+
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"foo = A\n"
+"bar = B\n"
+"pool A\n"
+" depth = 3\n"
+"pool C\n"
+" depth = 3\n"
+"rule echo\n"
+" command = replaced by next line\n"
+" command = echo $foo,$bar\n"
+" pool = $foo\n"
+"build a: echo\n"
+" bar = replaced by next line\n"
+" bar = edge:$foo\n"
+"foo = C\n"
+"bar = D\n"));
+
+ Edge* edge = GetNode("a")->in_edge();
+ EXPECT_EQ("echo C,edge:A", edge->GetBinding("command"));
+ EXPECT_EQ("A", edge->pool()->name());
+ EXPECT_EQ("C", edge->GetBinding("pool"));
+}
+
+TEST_F(GraphTest, PhonyOutputAlwaysDirty) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"rule phony_out\n"
+" command = echo ${out}\n"
+" phony_output = true\n"
+"build foo: phony_out\n"));
+
+ fs_.Create("foo", "");
+ string err;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("foo"), NULL, &err));
+ ASSERT_EQ("", err);
+
+ EXPECT_TRUE(GetNode("foo")->dirty());
+}
+
+TEST_F(GraphTest, Validation) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"build out: cat in |@ validate\n"
+"build validate: cat in\n"));
+
+ fs_.Create("in", "");
+ string err;
+ std::vector<Node*> validation_nodes;
+ EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &validation_nodes, &err));
+ ASSERT_EQ("", err);
+
+ ASSERT_EQ(validation_nodes.size(), 1);
+ EXPECT_EQ(validation_nodes[0]->path(), "validate");
+
+ EXPECT_TRUE(GetNode("out")->dirty());
+ EXPECT_TRUE(GetNode("validate")->dirty());
+}
diff --git a/src/graphviz.h b/src/graphviz.h
index 408496d..6dd08be 100644
--- a/src/graphviz.h
+++ b/src/graphviz.h
@@ -17,6 +17,8 @@
#include <set>
+#include "graph.h"
+
struct Node;
struct Edge;
@@ -27,7 +29,7 @@
void Finish();
std::set<Node*> visited_nodes_;
- std::set<Edge*> visited_edges_;
+ EdgeSet visited_edges_;
};
#endif // NINJA_GRAPHVIZ_H_
diff --git a/src/hash_map.h b/src/hash_map.h
index 55d2c9d..717a7c9 100644
--- a/src/hash_map.h
+++ b/src/hash_map.h
@@ -12,12 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#ifndef NINJA_MAP_H_
-#define NINJA_MAP_H_
+#ifndef NINJA_HASH_MAP_H_
+#define NINJA_HASH_MAP_H_
-#include <algorithm>
#include <string.h>
-#include "string_piece.h"
#include "util.h"
// MurmurHash2, by Austin Appleby
@@ -53,71 +51,4 @@
return h;
}
-#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)
-#include <unordered_map>
-
-namespace std {
-template<>
-struct hash<StringPiece> {
- typedef StringPiece argument_type;
- typedef size_t result_type;
-
- size_t operator()(StringPiece key) const {
- return MurmurHash2(key.str_, key.len_);
- }
-};
-}
-
-#elif defined(_MSC_VER)
-#include <hash_map>
-
-using stdext::hash_map;
-using stdext::hash_compare;
-
-struct StringPieceCmp : public hash_compare<StringPiece> {
- size_t operator()(const StringPiece& key) const {
- return MurmurHash2(key.str_, key.len_);
- }
- bool operator()(const StringPiece& a, const StringPiece& b) const {
- int cmp = memcmp(a.str_, b.str_, min(a.len_, b.len_));
- if (cmp < 0) {
- return true;
- } else if (cmp > 0) {
- return false;
- } else {
- return a.len_ < b.len_;
- }
- }
-};
-
-#else
-#include <ext/hash_map>
-
-using __gnu_cxx::hash_map;
-
-namespace __gnu_cxx {
-template<>
-struct hash<StringPiece> {
- size_t operator()(StringPiece key) const {
- return MurmurHash2(key.str_, key.len_);
- }
-};
-}
-#endif
-
-/// A template for hash_maps keyed by a StringPiece whose string is
-/// owned externally (typically by the values). Use like:
-/// ExternalStringHash<Foo*>::Type foos; to make foos into a hash
-/// mapping StringPiece => Foo*.
-template<typename V>
-struct ExternalStringHashMap {
-#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900)
- typedef std::unordered_map<StringPiece, V> Type;
-#elif defined(_MSC_VER)
- typedef hash_map<StringPiece, V, StringPieceCmp> Type;
-#else
- typedef hash_map<StringPiece, V> Type;
-#endif
-};
-
-#endif // NINJA_MAP_H_
+#endif // NINJA_HASH_MAP_H_
diff --git a/src/hashed_str_view.h b/src/hashed_str_view.h
new file mode 100644
index 0000000..ea19a4f
--- /dev/null
+++ b/src/hashed_str_view.h
@@ -0,0 +1,163 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_HASHED_STR_VIEW_
+#define NINJA_HASHED_STR_VIEW_
+
+#include <stdint.h>
+
+#include "string_piece.h"
+
+class HashedStr;
+
+using StrHashType = size_t;
+
+class HashedStrView {
+public:
+ constexpr HashedStrView() {}
+
+ HashedStrView(const char* str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStrView(const std::string& str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStrView(StringPiece str) : str_(str), hash_(HashStr(str_)) {}
+ inline HashedStrView(const HashedStr& str);
+
+ // Bypass the hashing step when necessary for better performance.
+ explicit HashedStrView(const StringPiece& str, StrHashType hash)
+ : str_(str), hash_(hash) {}
+
+ const StringPiece& str_view() const { return str_; }
+
+ StrHashType hash() const { return hash_; }
+ const char* data() const { return str_.data(); }
+ size_t size() const { return str_.size(); }
+ bool empty() const { return str_.empty(); }
+
+private:
+ // Reversing the order of these fields would break the constructors, most of
+ // which initialize hash_ using str_.
+ StringPiece str_;
+ StrHashType hash_ = 0;
+};
+
+class HashedStr {
+public:
+ HashedStr() {}
+
+ HashedStr(const char* str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStr(const std::string& str) : str_(str), hash_(HashStr(str_)) {}
+ HashedStr(std::string&& str) : str_(std::move(str)), hash_(HashStr(str_)) {}
+ explicit HashedStr(StringPiece str) : str_(NonNullData(str), str.size()), hash_(HashStr(str_)) {}
+ explicit HashedStr(const HashedStrView& str) : str_(NonNullData(str.str_view()), str.size()), hash_(str.hash()) {}
+
+ HashedStr& operator=(std::string&& str) {
+ str_ = std::move(str);
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(const std::string& str) {
+ str_ = str;
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(StringPiece str) {
+ str_.assign(NonNullData(str), str.size());
+ hash_ = HashStr(str_);
+ return *this;
+ }
+
+ HashedStr& operator=(const HashedStrView& str) {
+ str_.assign(NonNullData(str.str_view()), str.size());
+ hash_ = str.hash();
+ return *this;
+ }
+
+ const std::string& str() const { return str_; }
+
+ StrHashType hash() const { return hash_; }
+ const char* c_str() const { return str_.c_str(); }
+ const char* data() const { return str_.data(); }
+ size_t size() const { return str_.size(); }
+ bool empty() const { return str_.empty(); }
+
+private:
+ static const char* NonNullData(const StringPiece& piece) {
+ return piece.data() == nullptr ? "" : piece.data();
+ }
+
+ // Reversing the order of these fields would break the constructors, most of
+ // which initialize hash_ using str_.
+ std::string str_;
+ StrHashType hash_ = 0;
+};
+
+inline HashedStrView::HashedStrView(const HashedStr& str) : str_(str.str()), hash_(str.hash()) {}
+
+inline bool operator==(const HashedStr& x, const HashedStr& y) {
+ return x.hash() == y.hash() && x.str() == y.str();
+}
+
+inline bool operator==(const HashedStr& x, const HashedStrView& y) {
+ return x.hash() == y.hash() && x.str() == y.str_view();
+}
+
+inline bool operator==(const HashedStrView& x, const HashedStr& y) {
+ return x.hash() == y.hash() && x.str_view() == y.str();
+}
+
+inline bool operator==(const HashedStrView& x, const HashedStrView& y) {
+ return x.hash() == y.hash() && x.str_view() == y.str_view();
+}
+
+inline bool operator!=(const HashedStr& x, const HashedStr& y) { return !(x == y); }
+inline bool operator!=(const HashedStr& x, const HashedStrView& y) { return !(x == y); }
+inline bool operator!=(const HashedStrView& x, const HashedStr& y) { return !(x == y); }
+inline bool operator!=(const HashedStrView& x, const HashedStrView& y) { return !(x == y); }
+
+namespace std {
+ template<> struct hash<HashedStrView> {
+ size_t operator()(const HashedStrView& x) const { return x.hash(); }
+ };
+
+ template<> struct hash<HashedStr> {
+ size_t operator()(const HashedStr& x) const { return x.hash(); }
+ };
+}
+
+// The comparison operators ignore the hash code. For efficiency, the
+// ConcurrentHashMap orders keys by their hash code first, then by their
+// operator< functions.
+
+inline bool operator<(const HashedStr& x, const HashedStr& y) { return x.str() < y.str(); }
+inline bool operator>(const HashedStr& x, const HashedStr& y) { return x.str() > y.str(); }
+inline bool operator<=(const HashedStr& x, const HashedStr& y) { return x.str() <= y.str(); }
+inline bool operator>=(const HashedStr& x, const HashedStr& y) { return x.str() >= y.str(); }
+
+inline bool operator<(const HashedStr& x, const HashedStrView& y) { return x.str() < y.str_view(); }
+inline bool operator>(const HashedStr& x, const HashedStrView& y) { return x.str() > y.str_view(); }
+inline bool operator<=(const HashedStr& x, const HashedStrView& y) { return x.str() <= y.str_view(); }
+inline bool operator>=(const HashedStr& x, const HashedStrView& y) { return x.str() >= y.str_view(); }
+
+inline bool operator<(const HashedStrView& x, const HashedStr& y) { return x.str_view() < y.str(); }
+inline bool operator>(const HashedStrView& x, const HashedStr& y) { return x.str_view() > y.str(); }
+inline bool operator<=(const HashedStrView& x, const HashedStr& y) { return x.str_view() <= y.str(); }
+inline bool operator>=(const HashedStrView& x, const HashedStr& y) { return x.str_view() >= y.str(); }
+
+inline bool operator<(const HashedStrView& x, const HashedStrView& y) { return x.str_view() < y.str_view(); }
+inline bool operator>(const HashedStrView& x, const HashedStrView& y) { return x.str_view() > y.str_view(); }
+inline bool operator<=(const HashedStrView& x, const HashedStrView& y) { return x.str_view() <= y.str_view(); }
+inline bool operator>=(const HashedStrView& x, const HashedStrView& y) { return x.str_view() >= y.str_view(); }
+
+#endif // NINJA_HASHED_STR_VIEW_
diff --git a/src/lexer.cc b/src/lexer.cc
index 35ae97b..a5c3c27 100644
--- a/src/lexer.cc
+++ b/src/lexer.cc
@@ -1,4 +1,4 @@
-/* Generated by re2c 0.16 */
+/* Generated by re2c 1.3 */
// Copyright 2011 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,24 +18,79 @@
#include <stdio.h>
#include "eval_env.h"
+#include "graph.h"
#include "util.h"
-bool Lexer::Error(const string& message, string* err) {
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx) {
+ assert(idx <= content.size());
+
+ // Iterate over each LF in the manifest, starting at the given index.
+ while (true) {
+ const void* next_line = memchr(content.data() + idx, '\n',
+ content.size() - idx);
+ if (next_line == nullptr) {
+ break;
+ }
+ idx = static_cast<const char*>(next_line) - content.data();
+ ++idx; // step over the LF
+
+ // The line must not be preceded by a line continuator. This logic can
+ // filter out more split candidates than strictly necessary:
+ // - The preceding line could have a comment that ends with a "$": "# $\n"
+ // - The preceding line could end with an escaped-dollar: "X=$$\n"
+ if ((idx >= 2 && content.substr(idx - 2, 2) == "$\n") ||
+ (idx >= 3 && content.substr(idx - 3, 3) == "$\r\n")) {
+ continue;
+ }
+
+ // Skip an indented line or a comment line, either of which could be part of
+ // an earlier declaration. Ninja allows unindented comments (as well as
+ // indented comments) inside a binding block, e.g.:
+ //
+ // build foo: cc
+ // # comment-line
+ // pool = link_pool
+ //
+ // Ninja doesn't allow blank lines in a binding block. This code could
+ // probably allow a chunk to start with a blank line, but it seems better if
+ // it doesn't.
+ if (idx >= content.size() ||
+ content[idx] == ' ' || content[idx] == '#' ||
+ content[idx] == '\r' || content[idx] == '\n') {
+ continue;
+ }
+
+ return idx;
+ }
+
+ return content.size();
+}
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err) {
+ // Make a copy in case message and err alias.
+ std::string message_tmp = message;
+
// Compute line/column.
int line = 1;
- const char* line_start = input_.str_;
- for (const char* p = input_.str_; p < last_token_; ++p) {
+ const char* line_start = file_start;
+ const char* file_pos = file_start + file_offset;
+
+ for (const char* p = line_start; p < file_pos; ++p) {
if (*p == '\n') {
++line;
line_start = p + 1;
}
}
- int col = last_token_ ? (int)(last_token_ - line_start) : 0;
+ int col = (int)(file_pos - line_start);
char buf[1024];
- snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename.c_str(), line);
*err = buf;
- *err += message + "\n";
+ *err += message_tmp + "\n";
// Add some context to the message.
const int kTruncateColumn = 72;
@@ -59,15 +114,16 @@
return false;
}
-Lexer::Lexer(const char* input) {
- Start("input", input);
+bool Lexer::Error(const std::string& message, std::string* err) {
+ return DecorateErrorWithLocation(filename_, input_.data(),
+ GetLastTokenOffset(), message, err);
}
-void Lexer::Start(StringPiece filename, StringPiece input) {
- filename_ = filename;
- input_ = input;
- ofs_ = input_.str_;
- last_token_ = NULL;
+bool Lexer::UnexpectedNulError(const char* pos, std::string* err) {
+ assert(*pos == '\0');
+ const char* msg = (pos == EndOfFile()) ? "unexpected EOF"
+ : "unexpected NUL byte";
+ return Error(msg, err);
}
const char* Lexer::TokenName(Token t) {
@@ -83,9 +139,11 @@
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case PIPEAT: return "'|@'";
case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
+ case TNUL: return "nul byte";
case TEOF: return "eof";
}
return NULL; // not reached
@@ -117,6 +175,7 @@
Lexer::Token Lexer::ReadToken() {
const char* p = ofs_;
const char* q;
+ const char* r;
const char* start;
Lexer::Token token;
for (;;) {
@@ -127,7 +186,7 @@
unsigned int yyaccept = 0;
static const unsigned char yybm[] = {
0, 128, 128, 128, 128, 128, 128, 128,
- 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
128, 128, 128, 128, 128, 128, 128, 128,
160, 128, 128, 128, 128, 128, 128, 128,
@@ -219,7 +278,7 @@
}
yy2:
++p;
- { token = TEOF; break; }
+ { token = (start == EndOfFile()) ? TEOF : TNUL; break; }
yy4:
++p;
yy5:
@@ -229,20 +288,19 @@
{ token = NEWLINE; break; }
yy8:
yych = *++p;
- if (yych == '\n') goto yy28;
+ if (yych == '\n') goto yy6;
goto yy5;
yy9:
yyaccept = 0;
- q = ++p;
- yych = *p;
+ yych = *(q = ++p);
if (yybm[0+yych] & 32) {
goto yy9;
}
if (yych <= '\f') {
if (yych == '\n') goto yy6;
} else {
- if (yych <= '\r') goto yy30;
- if (yych == '#') goto yy32;
+ if (yych <= '\r') goto yy28;
+ if (yych == '#') goto yy30;
}
yy11:
{ token = INDENT; break; }
@@ -250,10 +308,9 @@
yyaccept = 1;
yych = *(q = ++p);
if (yych <= 0x00) goto yy5;
- goto yy33;
+ goto yy31;
yy13:
- ++p;
- yych = *p;
+ yych = *++p;
yy14:
if (yybm[0+yych] & 64) {
goto yy13;
@@ -267,93 +324,100 @@
{ token = EQUALS; break; }
yy20:
yych = *++p;
- if (yych == 'u') goto yy36;
+ if (yych == 'u') goto yy35;
goto yy14;
yy21:
yych = *++p;
- if (yych == 'e') goto yy37;
+ if (yych == 'e') goto yy36;
goto yy14;
yy22:
yych = *++p;
- if (yych == 'n') goto yy38;
+ if (yych == 'n') goto yy37;
goto yy14;
yy23:
yych = *++p;
- if (yych == 'o') goto yy39;
+ if (yych == 'o') goto yy38;
goto yy14;
yy24:
yych = *++p;
- if (yych == 'u') goto yy40;
+ if (yych == 'u') goto yy39;
goto yy14;
yy25:
yych = *++p;
- if (yych == 'u') goto yy41;
+ if (yych == 'u') goto yy40;
goto yy14;
yy26:
- ++p;
- if ((yych = *p) == '|') goto yy42;
+ yych = *++p;
+ if (yych == '@') goto yy41;
+ if (yych == '|') goto yy43;
{ token = PIPE; break; }
yy28:
- ++p;
- { token = NEWLINE; break; }
-yy30:
yych = *++p;
- if (yych == '\n') goto yy28;
-yy31:
+ if (yych == '\n') goto yy6;
+yy29:
p = q;
if (yyaccept == 0) {
goto yy11;
} else {
goto yy5;
}
+yy30:
+ yych = *++p;
+yy31:
+ if (yybm[0+yych] & 128) {
+ goto yy30;
+ }
+ if (yych <= 0x00) goto yy29;
+ if (yych >= '\v') {
+ r = p;
+ goto yy34;
+ }
+ r = p;
yy32:
++p;
- yych = *p;
-yy33:
- if (yybm[0+yych] & 128) {
- goto yy32;
- }
- if (yych <= 0x00) goto yy31;
- ++p;
+ p = r;
{ continue; }
+yy34:
+ yych = *++p;
+ if (yych == '\n') goto yy32;
+ goto yy29;
+yy35:
+ yych = *++p;
+ if (yych == 'i') goto yy45;
+ goto yy14;
yy36:
yych = *++p;
- if (yych == 'i') goto yy44;
+ if (yych == 'f') goto yy46;
goto yy14;
yy37:
yych = *++p;
- if (yych == 'f') goto yy45;
+ if (yych == 'c') goto yy47;
goto yy14;
yy38:
yych = *++p;
- if (yych == 'c') goto yy46;
+ if (yych == 'o') goto yy48;
goto yy14;
yy39:
yych = *++p;
- if (yych == 'o') goto yy47;
+ if (yych == 'l') goto yy49;
goto yy14;
yy40:
yych = *++p;
- if (yych == 'l') goto yy48;
+ if (yych == 'b') goto yy50;
goto yy14;
yy41:
- yych = *++p;
- if (yych == 'b') goto yy49;
- goto yy14;
-yy42:
+ ++p;
+ { token = PIPEAT; break; }
+yy43:
++p;
{ token = PIPE2; break; }
-yy44:
- yych = *++p;
- if (yych == 'l') goto yy50;
- goto yy14;
yy45:
yych = *++p;
- if (yych == 'a') goto yy51;
+ if (yych == 'l') goto yy51;
goto yy14;
yy46:
yych = *++p;
- if (yych == 'l') goto yy52;
+ if (yych == 'a') goto yy52;
goto yy14;
yy47:
yych = *++p;
@@ -361,87 +425,91 @@
goto yy14;
yy48:
yych = *++p;
- if (yych == 'e') goto yy55;
+ if (yych == 'l') goto yy54;
goto yy14;
yy49:
yych = *++p;
- if (yych == 'n') goto yy57;
+ if (yych == 'e') goto yy56;
goto yy14;
yy50:
yych = *++p;
- if (yych == 'd') goto yy58;
+ if (yych == 'n') goto yy58;
goto yy14;
yy51:
yych = *++p;
- if (yych == 'u') goto yy60;
+ if (yych == 'd') goto yy59;
goto yy14;
yy52:
yych = *++p;
if (yych == 'u') goto yy61;
goto yy14;
yy53:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'u') goto yy62;
+ goto yy14;
+yy54:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = POOL; break; }
-yy55:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+yy56:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = RULE; break; }
-yy57:
- yych = *++p;
- if (yych == 'i') goto yy62;
- goto yy14;
yy58:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'i') goto yy63;
+ goto yy14;
+yy59:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = BUILD; break; }
-yy60:
- yych = *++p;
- if (yych == 'l') goto yy63;
- goto yy14;
yy61:
yych = *++p;
- if (yych == 'd') goto yy64;
+ if (yych == 'l') goto yy64;
goto yy14;
yy62:
yych = *++p;
- if (yych == 'n') goto yy65;
+ if (yych == 'd') goto yy65;
goto yy14;
yy63:
yych = *++p;
- if (yych == 't') goto yy66;
+ if (yych == 'n') goto yy66;
goto yy14;
yy64:
yych = *++p;
- if (yych == 'e') goto yy68;
+ if (yych == 't') goto yy67;
goto yy14;
yy65:
yych = *++p;
- if (yych == 'j') goto yy70;
+ if (yych == 'e') goto yy69;
goto yy14;
yy66:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yych == 'j') goto yy71;
+ goto yy14;
+yy67:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = DEFAULT; break; }
-yy68:
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+yy69:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = INCLUDE; break; }
-yy70:
+yy71:
yych = *++p;
if (yych != 'a') goto yy14;
- ++p;
- if (yybm[0+(yych = *p)] & 64) {
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
goto yy13;
}
{ token = SUBNINJA; break; }
@@ -456,6 +524,119 @@
return token;
}
+bool Lexer::PeekIndent() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+
+{
+ unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 192, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 64) {
+ goto yy81;
+ }
+ if (yych <= '\f') {
+ if (yych == '\n') goto yy78;
+ } else {
+ if (yych <= '\r') goto yy80;
+ if (yych == '#') goto yy84;
+ }
+ ++p;
+yy77:
+ { last_token_ = ofs_ = start; return false; }
+yy78:
+ ++p;
+ { last_token_ = ofs_ = start; return false; }
+yy80:
+ yych = *++p;
+ if (yych == '\n') goto yy78;
+ goto yy77;
+yy81:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy81;
+ }
+ if (yych <= '\f') {
+ if (yych == '\n') goto yy78;
+ } else {
+ if (yych <= '\r') goto yy85;
+ if (yych == '#') goto yy87;
+ }
+yy83:
+ { last_token_ = start; ofs_ = p; return true; }
+yy84:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yych <= 0x00) goto yy77;
+ goto yy88;
+yy85:
+ yych = *++p;
+ if (yych == '\n') goto yy78;
+yy86:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy83;
+ } else {
+ goto yy77;
+ }
+yy87:
+ yych = *++p;
+yy88:
+ if (yybm[0+yych] & 128) {
+ goto yy87;
+ }
+ if (yych <= 0x00) goto yy86;
+ if (yych >= '\v') goto yy91;
+yy89:
+ ++p;
+ { continue; }
+yy91:
+ yych = *++p;
+ if (yych == '\n') goto yy89;
+ goto yy86;
+}
+
+ }
+}
+
bool Lexer::PeekToken(Token token) {
Token t = ReadToken();
if (t == token)
@@ -508,47 +689,43 @@
};
yych = *p;
if (yybm[0+yych] & 128) {
- goto yy79;
+ goto yy98;
}
- if (yych <= 0x00) goto yy75;
- if (yych == '$') goto yy82;
- goto yy77;
-yy75:
+ if (yych <= 0x00) goto yy94;
+ if (yych == '$') goto yy101;
+ goto yy96;
+yy94:
++p;
{ break; }
-yy77:
+yy96:
++p;
-yy78:
+yy97:
{ break; }
-yy79:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 128) {
- goto yy79;
- }
- { continue; }
-yy82:
- yych = *(q = ++p);
- if (yych == '\n') goto yy83;
- if (yych == '\r') goto yy85;
- goto yy78;
-yy83:
- ++p;
- { continue; }
-yy85:
+yy98:
yych = *++p;
- if (yych == '\n') goto yy87;
- p = q;
- goto yy78;
-yy87:
+ if (yybm[0+yych] & 128) {
+ goto yy98;
+ }
+ { continue; }
+yy101:
+ yych = *(q = ++p);
+ if (yych == '\n') goto yy102;
+ if (yych == '\r') goto yy104;
+ goto yy97;
+yy102:
++p;
{ continue; }
+yy104:
+ yych = *++p;
+ if (yych == '\n') goto yy102;
+ p = q;
+ goto yy97;
}
}
}
-bool Lexer::ReadIdent(string* out) {
+bool Lexer::ReadIdent(StringPiece* out) {
const char* p = ofs_;
const char* start;
for (;;) {
@@ -592,21 +769,20 @@
};
yych = *p;
if (yybm[0+yych] & 128) {
- goto yy93;
+ goto yy110;
}
++p;
{
last_token_ = start;
return false;
}
-yy93:
- ++p;
- yych = *p;
+yy110:
+ yych = *++p;
if (yybm[0+yych] & 128) {
- goto yy93;
+ goto yy110;
}
{
- out->assign(start, p - start);
+ *out = StringPiece(start, p - start);
break;
}
}
@@ -618,7 +794,7 @@
return true;
}
-bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+bool Lexer::ReadBindingValue(StringPiece* out, string* err) {
const char* p = ofs_;
const char* q;
const char* start;
@@ -627,6 +803,1126 @@
{
unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 0, 64, 64, 0, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 0, 64, 64, 64,
+ 64, 64, 64, 64, 64, 192, 192, 64,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 64, 64, 64, 64, 64, 64,
+ 64, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 64, 64, 64, 64, 192,
+ 64, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 192, 192, 192, 192, 192,
+ 192, 192, 192, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ 64, 64, 64, 64, 64, 64, 64, 64,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 64) {
+ goto yy117;
+ }
+ if (yych <= 0x00) goto yy115;
+ if (yych <= '\n') goto yy120;
+ if (yych <= '\r') goto yy122;
+ goto yy124;
+yy115:
+ ++p;
+ {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+yy117:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy117;
+ }
+ if (yych >= 0x0E) goto yy125;
+yy119:
+ {
+ continue;
+ }
+yy120:
+ ++p;
+ {
+ break;
+ }
+yy122:
+ yych = *++p;
+ if (yych == '\n') goto yy120;
+ {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+yy124:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych <= '\t') goto yy127;
+ goto yy117;
+ } else {
+ if (yych == '\r') goto yy129;
+ goto yy127;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy117;
+ goto yy127;
+ } else {
+ if (yych <= '$') goto yy117;
+ if (yych <= ',') goto yy127;
+ goto yy117;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych <= '/') goto yy127;
+ goto yy117;
+ } else {
+ if (yych <= '@') goto yy127;
+ if (yych <= 'Z') goto yy117;
+ goto yy127;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy117;
+ goto yy127;
+ } else {
+ if (yych <= 'z') goto yy117;
+ if (yych <= '{') goto yy130;
+ goto yy127;
+ }
+ }
+ }
+yy125:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych >= '\n') goto yy117;
+ } else {
+ if (yych == '\r') goto yy131;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy117;
+ } else {
+ if (yych <= '$') goto yy117;
+ if (yych >= '-') goto yy117;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych >= '0') goto yy117;
+ } else {
+ if (yych <= '@') goto yy126;
+ if (yych <= 'Z') goto yy117;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy117;
+ } else {
+ if (yych <= 'z') goto yy117;
+ if (yych <= '{') goto yy132;
+ }
+ }
+ }
+yy126:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy119;
+ } else {
+ goto yy128;
+ }
+yy127:
+ ++p;
+yy128:
+ {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+yy129:
+ yych = *++p;
+ if (yych == '\n') goto yy117;
+ goto yy128;
+yy130:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ goto yy128;
+yy131:
+ yych = *++p;
+ if (yych == '\n') goto yy117;
+ goto yy126;
+yy132:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ goto yy126;
+yy133:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy133;
+ }
+ if (yych == '}') goto yy117;
+ goto yy126;
+}
+
+ }
+ *out = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ // Non-path strings end in newlines, so there's no whitespace to eat.
+ return true;
+}
+
+StringPiece Lexer::PeekCanonicalPath() {
+ auto finish = [this](const char* start, const char* end) {
+ ofs_ = end;
+ EatWhitespace();
+ return StringPiece(start, end - start);
+ };
+
+ const char* p = ofs_;
+ const char* q;
+ const char* r;
+ last_token_ = ofs_;
+
+ do {
+
+{
+ unsigned char yych;
+ static const unsigned char yybm[] = {
+ 0, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 0, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 0, 128, 128, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 0,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 0, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 0, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ 128, 128, 128, 128, 128, 128, 128, 128,
+ };
+ yych = *p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy137;
+ if (yych != '\n') goto yy139;
+ } else {
+ if (yych <= '\r') goto yy137;
+ if (yych != ' ') goto yy139;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy137;
+ if (yych <= '-') goto yy139;
+ if (yych <= '.') goto yy140;
+ goto yy141;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy139;
+ } else {
+ if (yych != '|') goto yy139;
+ }
+ }
+ }
+yy137:
+ ++p;
+yy138:
+ { break; }
+yy139:
+ yych = *(q = ++p);
+ if (yych <= 0x00) goto yy138;
+ if (yych == '$') goto yy138;
+ goto yy143;
+yy140:
+ yych = *(q = ++p);
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy138;
+ if (yych == '\n') goto yy138;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy138;
+ if (yych == ' ') goto yy138;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy138;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy149;
+ goto yy150;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy138;
+ } else {
+ if (yych == '|') goto yy138;
+ goto yy142;
+ }
+ }
+ }
+yy141:
+ yych = *(q = ++p);
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy138;
+ if (yych == '\n') goto yy138;
+ } else {
+ if (yych <= '\r') goto yy138;
+ if (yych == ' ') goto yy138;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy138;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy151;
+ goto yy138;
+ } else {
+ if (yych <= ':') {
+ if (yych >= ':') goto yy138;
+ } else {
+ if (yych == '|') goto yy138;
+ }
+ }
+ }
+yy142:
+ yych = *++p;
+yy143:
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ if (yych <= '\r') {
+ if (yych <= 0x00) goto yy144;
+ if (yych <= '\n') {
+ r = p;
+ goto yy145;
+ }
+ r = p;
+ goto yy147;
+ } else {
+ if (yych <= ' ') {
+ r = p;
+ goto yy145;
+ }
+ if (yych <= '$') goto yy144;
+ if (yych <= '/') goto yy148;
+ r = p;
+ goto yy145;
+ }
+yy144:
+ p = q;
+ goto yy138;
+yy145:
+ ++p;
+ p = r;
+ { return finish(ofs_, p); }
+yy147:
+ yych = *++p;
+ if (yych == '\n') goto yy145;
+ goto yy144;
+yy148:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy151;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy149:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ if (yych <= '$') goto yy144;
+ if (yych <= '/') goto yy152;
+ goto yy144;
+yy150:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy155;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy151:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy156;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy152:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy157;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy153:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ } else {
+ if (yych <= '\r') {
+ r = p;
+ goto yy160;
+ }
+ if (yych == ' ') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy161;
+ goto yy153;
+ } else {
+ if (yych <= ':') {
+ r = p;
+ goto yy158;
+ }
+ if (yych == '|') {
+ r = p;
+ goto yy158;
+ }
+ goto yy153;
+ }
+ }
+yy155:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy162;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy156:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy142;
+ }
+ goto yy144;
+yy157:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy142;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy142;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy142;
+ if (yych <= '.') goto yy149;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy142;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy142;
+ }
+ }
+ }
+yy158:
+ ++p;
+ p = r;
+ { return finish(ofs_ + 2, p); }
+yy160:
+ yych = *++p;
+ if (yych == '\n') goto yy158;
+ goto yy144;
+yy161:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych <= '.') goto yy163;
+ goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+yy162:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy150;
+ goto yy153;
+ } else {
+ if (yych <= ':') goto yy144;
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+yy163:
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych <= '$') goto yy144;
+ if (yych <= '-') goto yy153;
+ if (yych >= '/') goto yy144;
+ } else {
+ if (yych <= ':') {
+ if (yych <= '9') goto yy153;
+ goto yy144;
+ } else {
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+ }
+ yych = *++p;
+ if (yych <= '#') {
+ if (yych <= '\f') {
+ if (yych <= 0x00) goto yy144;
+ if (yych == '\n') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= '\r') goto yy144;
+ if (yych == ' ') goto yy144;
+ goto yy153;
+ }
+ } else {
+ if (yych <= '9') {
+ if (yych <= '$') goto yy144;
+ if (yych == '/') goto yy144;
+ goto yy153;
+ } else {
+ if (yych <= ':') goto yy144;
+ if (yych == '|') goto yy144;
+ goto yy153;
+ }
+ }
+}
+
+ } while (false);
+
+ return {};
+}
+
+bool Lexer::ReadPath(LexedPath* out, std::string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+
+{
+ unsigned char yych;
+ unsigned int yyaccept = 0;
+ static const unsigned char yybm[] = {
+ 0, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 0, 32, 32, 0, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 64, 32, 32, 32, 0, 32, 32, 32,
+ 32, 32, 32, 32, 32, 160, 160, 32,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 0, 32, 32, 32, 32, 32,
+ 32, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 32, 32, 32, 32, 160,
+ 32, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 160, 160, 160, 160, 160,
+ 160, 160, 160, 32, 0, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= '\r') {
+ if (yych <= 0x00) goto yy167;
+ if (yych <= '\n') goto yy172;
+ goto yy174;
+ } else {
+ if (yych <= ' ') goto yy172;
+ if (yych <= '$') goto yy176;
+ goto yy172;
+ }
+yy167:
+ ++p;
+ {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+yy169:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= ' ') goto yy171;
+ if (yych <= '$') goto yy177;
+yy171:
+ {
+ continue;
+ }
+yy172:
+ ++p;
+ {
+ p = start;
+ break;
+ }
+yy174:
+ yych = *++p;
+ if (yych == '\n') goto yy172;
+ {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+yy176:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych <= '\t') goto yy179;
+ goto yy181;
+ } else {
+ if (yych == '\r') goto yy183;
+ goto yy179;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy169;
+ goto yy179;
+ } else {
+ if (yych <= '$') goto yy169;
+ if (yych <= ',') goto yy179;
+ goto yy169;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych <= '/') goto yy179;
+ goto yy169;
+ } else {
+ if (yych <= '@') goto yy179;
+ if (yych <= 'Z') goto yy169;
+ goto yy179;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy169;
+ goto yy179;
+ } else {
+ if (yych <= 'z') goto yy169;
+ if (yych <= '{') goto yy184;
+ goto yy179;
+ }
+ }
+ }
+yy177:
+ yych = *++p;
+ if (yych <= '-') {
+ if (yych <= 0x1F) {
+ if (yych <= '\n') {
+ if (yych >= '\n') goto yy181;
+ } else {
+ if (yych == '\r') goto yy185;
+ }
+ } else {
+ if (yych <= '#') {
+ if (yych <= ' ') goto yy169;
+ } else {
+ if (yych <= '$') goto yy169;
+ if (yych >= '-') goto yy169;
+ }
+ }
+ } else {
+ if (yych <= '^') {
+ if (yych <= ':') {
+ if (yych >= '0') goto yy169;
+ } else {
+ if (yych <= '@') goto yy178;
+ if (yych <= 'Z') goto yy169;
+ }
+ } else {
+ if (yych <= '`') {
+ if (yych <= '_') goto yy169;
+ } else {
+ if (yych <= 'z') goto yy169;
+ if (yych <= '{') goto yy186;
+ }
+ }
+ }
+yy178:
+ p = q;
+ if (yyaccept == 0) {
+ goto yy171;
+ } else {
+ goto yy180;
+ }
+yy179:
+ ++p;
+yy180:
+ {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+yy181:
+ yyaccept = 0;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 32) {
+ goto yy169;
+ }
+ if (yych <= '\r') goto yy171;
+ if (yych <= ' ') goto yy181;
+ if (yych <= '$') goto yy177;
+ goto yy171;
+yy183:
+ yych = *++p;
+ if (yych == '\n') goto yy181;
+ goto yy180;
+yy184:
+ yyaccept = 1;
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ goto yy180;
+yy185:
+ yych = *++p;
+ if (yych == '\n') goto yy181;
+ goto yy178;
+yy186:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ goto yy178;
+yy187:
+ yych = *++p;
+ if (yybm[0+yych] & 128) {
+ goto yy187;
+ }
+ if (yych == '}') goto yy169;
+ goto yy178;
+}
+
+ }
+ *out = {};
+ out->str_ = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+/// Append the let binding's evaluated value to the output string. The input
+/// StringPiece must include a valid binding terminator.
+template <typename EvalVar>
+static inline void EvaluateBinding(std::string* out_append, StringPiece value,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = value.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+
+{
+ unsigned char yych;
+ static const unsigned char yybm[] = {
+ 0, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 0, 16, 16, 0, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 48, 16, 16, 16, 0, 16, 16, 16,
+ 16, 16, 16, 16, 16, 208, 144, 16,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 16, 16, 16, 16, 16, 16,
+ 16, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 16, 16, 16, 16, 208,
+ 16, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 208, 208, 208, 208, 208,
+ 208, 208, 208, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ 16, 16, 16, 16, 16, 16, 16, 16,
+ };
+ yych = *p;
+ if (yybm[0+yych] & 16) {
+ goto yy193;
+ }
+ if (yych <= 0x00) goto yy191;
+ if (yych <= '\n') goto yy196;
+ if (yych <= '\r') goto yy198;
+ goto yy199;
+yy191:
+ ++p;
+yy192:
+ { assert(false && "bad input in EvaluateBinding"); abort(); }
+yy193:
+ yych = *++p;
+ if (yybm[0+yych] & 16) {
+ goto yy193;
+ }
+ { out_append->append(start, p - start); continue; }
+yy196:
+ ++p;
+ { break; }
+yy198:
+ yych = *++p;
+ if (yych == '\n') goto yy196;
+ goto yy192;
+yy199:
+ yych = *(q = ++p);
+ if (yybm[0+yych] & 64) {
+ goto yy207;
+ }
+ if (yych <= ' ') {
+ if (yych <= '\f') {
+ if (yych != '\n') goto yy192;
+ } else {
+ if (yych <= '\r') goto yy203;
+ if (yych <= 0x1F) goto yy192;
+ goto yy205;
+ }
+ } else {
+ if (yych <= '/') {
+ if (yych == '$') goto yy205;
+ goto yy192;
+ } else {
+ if (yych <= ':') goto yy205;
+ if (yych <= '`') goto yy192;
+ if (yych <= '{') goto yy210;
+ goto yy192;
+ }
+ }
+yy200:
+ yych = *++p;
+ if (yybm[0+yych] & 32) {
+ goto yy200;
+ }
+ { continue; }
+yy203:
+ yych = *++p;
+ if (yych == '\n') goto yy200;
+yy204:
+ p = q;
+ goto yy192;
+yy205:
+ ++p;
+ { out_append->push_back(start[1]); continue; }
+yy207:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
+ goto yy207;
+ }
+ { expand(start + 1, p); continue; }
+yy210:
+ yych = *++p;
+ if (yych == '}') goto yy204;
+ goto yy212;
+yy211:
+ yych = *++p;
+yy212:
+ if (yybm[0+yych] & 128) {
+ goto yy211;
+ }
+ if (yych != '}') goto yy204;
+ ++p;
+ { expand(start + 2, p - 1); continue; }
+}
+
+ }
+ assert((p == value.data() + value.size()) &&
+ "bad end pos in EvaluateBinding");
+}
+
+void EvaluateBindingInScope(std::string* out_append, StringPiece value,
+ ScopePosition pos) {
+ EvaluateBinding(out_append, value,
+ [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+bool EvaluateBindingOnRule(std::string* out_append, StringPiece value,
+ EdgeEval* edge_eval, std::string* err) {
+ bool result = true;
+ EvaluateBinding(out_append, value,
+ [out_append, edge_eval, &result, err](const HashedStrView& var) {
+ result = result && edge_eval->EvaluateVariable(out_append, var, err);
+ });
+ return result;
+}
+
+std::string EvaluateBindingForTesting(StringPiece value) {
+ std::string result;
+ EvaluateBinding(&result, value, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
+
+/// Append an evaluated path to the output string.
+///
+/// This function does not canonicalize the output. Ninja canonicalizes paths for
+/// build nodes, but not all paths (e.g. It doesn't canonicalize paths to
+/// included ninja files.)
+template <typename EvalVar>
+static inline void EvaluatePath(std::string* out_append, const LexedPath& path,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = path.str_.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+
+{
+ unsigned char yych;
static const unsigned char yybm[] = {
0, 16, 16, 16, 16, 16, 16, 16,
16, 16, 0, 16, 16, 0, 16, 16,
@@ -663,168 +1959,121 @@
};
yych = *p;
if (yybm[0+yych] & 16) {
- goto yy100;
+ goto yy219;
}
if (yych <= '\r') {
- if (yych <= 0x00) goto yy98;
- if (yych <= '\n') goto yy103;
- goto yy105;
+ if (yych <= 0x00) goto yy217;
+ if (yych <= '\n') goto yy222;
+ goto yy224;
} else {
- if (yych <= ' ') goto yy103;
- if (yych <= '$') goto yy107;
- goto yy103;
+ if (yych <= ' ') goto yy222;
+ if (yych <= '$') goto yy225;
+ goto yy222;
}
-yy98:
+yy217:
++p;
- {
- last_token_ = start;
- return Error("unexpected EOF", err);
- }
-yy100:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 16) {
- goto yy100;
- }
- {
- eval->AddText(StringPiece(start, p - start));
- continue;
- }
-yy103:
- ++p;
- {
- if (path) {
- p = start;
- break;
- } else {
- if (*start == '\n')
- break;
- eval->AddText(StringPiece(start, 1));
- continue;
- }
- }
-yy105:
- ++p;
- if ((yych = *p) == '\n') goto yy108;
- {
- last_token_ = start;
- return Error(DescribeLastError(), err);
- }
-yy107:
+yy218:
+ { assert(false && "bad input in EvaluatePath"); abort(); }
+yy219:
yych = *++p;
+ if (yybm[0+yych] & 16) {
+ goto yy219;
+ }
+ { out_append->append(start, p - start); continue; }
+yy222:
+ ++p;
+ { p = start; break; }
+yy224:
+ yych = *++p;
+ if (yych == '\n') goto yy222;
+ goto yy218;
+yy225:
+ yych = *(q = ++p);
if (yybm[0+yych] & 64) {
- goto yy120;
+ goto yy233;
}
if (yych <= ' ') {
if (yych <= '\f') {
- if (yych == '\n') goto yy112;
- goto yy110;
+ if (yych != '\n') goto yy218;
} else {
- if (yych <= '\r') goto yy115;
- if (yych <= 0x1F) goto yy110;
- goto yy116;
+ if (yych <= '\r') goto yy229;
+ if (yych <= 0x1F) goto yy218;
+ goto yy231;
}
} else {
if (yych <= '/') {
- if (yych == '$') goto yy118;
- goto yy110;
+ if (yych == '$') goto yy231;
+ goto yy218;
} else {
- if (yych <= ':') goto yy123;
- if (yych <= '`') goto yy110;
- if (yych <= '{') goto yy125;
- goto yy110;
+ if (yych <= ':') goto yy231;
+ if (yych <= '`') goto yy218;
+ if (yych <= '{') goto yy236;
+ goto yy218;
}
}
-yy108:
- ++p;
- {
- if (path)
- p = start;
- break;
- }
-yy110:
- ++p;
-yy111:
- {
- last_token_ = start;
- return Error("bad $-escape (literal $ must be written as $$)", err);
- }
-yy112:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 32) {
- goto yy112;
- }
- {
- continue;
- }
-yy115:
+yy226:
yych = *++p;
- if (yych == '\n') goto yy126;
- goto yy111;
-yy116:
- ++p;
- {
- eval->AddText(StringPiece(" ", 1));
- continue;
- }
-yy118:
- ++p;
- {
- eval->AddText(StringPiece("$", 1));
- continue;
- }
-yy120:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 64) {
- goto yy120;
+ if (yybm[0+yych] & 32) {
+ goto yy226;
}
- {
- eval->AddSpecial(StringPiece(start + 1, p - start - 1));
- continue;
- }
-yy123:
- ++p;
- {
- eval->AddText(StringPiece(":", 1));
- continue;
- }
-yy125:
- yych = *(q = ++p);
- if (yybm[0+yych] & 128) {
- goto yy129;
- }
- goto yy111;
-yy126:
- ++p;
- yych = *p;
- if (yych == ' ') goto yy126;
- {
- continue;
- }
-yy129:
- ++p;
- yych = *p;
- if (yybm[0+yych] & 128) {
- goto yy129;
- }
- if (yych == '}') goto yy132;
+ { continue; }
+yy229:
+ yych = *++p;
+ if (yych == '\n') goto yy226;
+yy230:
p = q;
- goto yy111;
-yy132:
+ goto yy218;
+yy231:
++p;
- {
- eval->AddSpecial(StringPiece(start + 2, p - start - 3));
- continue;
- }
+ { out_append->push_back(start[1]); continue; }
+yy233:
+ yych = *++p;
+ if (yybm[0+yych] & 64) {
+ goto yy233;
+ }
+ { expand(start + 1, p); continue; }
+yy236:
+ yych = *++p;
+ if (yych == '}') goto yy230;
+ goto yy238;
+yy237:
+ yych = *++p;
+yy238:
+ if (yybm[0+yych] & 128) {
+ goto yy237;
+ }
+ if (yych != '}') goto yy230;
+ ++p;
+ { expand(start + 2, p - 1); continue; }
}
}
- last_token_ = start;
- ofs_ = p;
- if (path)
- EatWhitespace();
- // Non-path strings end in newlines, so there's no whitespace to eat.
- return true;
+ assert((p == path.str_.data() + path.str_.size()) &&
+ "bad end pos in EvaluatePath");
+}
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos) {
+ EvaluatePath(out_append, path, [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge) {
+ EvaluatePath(out_append, path, [out_append, &edge](const HashedStrView& var) {
+ // First look for a binding on the edge itself, then fall back to the
+ // edge's enclosing scope.
+ if (edge.EvaluateVariableSelfOnly(out_append, var))
+ return;
+ Scope::EvaluateVariableAtPos(out_append, var, edge.pos_.scope_pos());
+ });
+}
+
+std::string EvaluatePathForTesting(const LexedPath& path) {
+ std::string result;
+ EvaluatePath(&result, path, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
}
diff --git a/src/lexer.h b/src/lexer.h
index f366556..42581a7 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -15,19 +15,46 @@
#ifndef NINJA_LEXER_H_
#define NINJA_LEXER_H_
+#include <assert.h>
+
+#include <string>
+
#include "string_piece.h"
+#include "util.h"
// Windows may #define ERROR.
#ifdef ERROR
#undef ERROR
#endif
+struct Edge;
+struct EdgeEval;
struct EvalString;
+struct LexedPath;
+struct ScopePosition;
+
+/// Search the manifest for the next acceptable point to split the manifest.
+/// Assuming the manifest is syntactically valid to the returned position, the
+/// chunk parser will finish parsing a declaration just before the returned
+/// position.
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx);
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err);
struct Lexer {
- Lexer() {}
+ /// Ordinary lexer constructor.
+ Lexer(const std::string& filename, StringPiece file_content, const char* pos)
+ : filename_(filename), input_(file_content), ofs_(pos) {
+ assert(pos >= &file_content[0] &&
+ pos <= file_content.data() + file_content.size());
+ }
+
/// Helper ctor useful for tests.
- explicit Lexer(const char* input);
+ explicit Lexer(const char* input) : Lexer("input", input, input) {}
enum Token {
ERROR,
@@ -41,9 +68,11 @@
NEWLINE,
PIPE,
PIPE2,
+ PIPEAT,
POOL,
RULE,
SUBNINJA,
+ TNUL,
TEOF,
};
@@ -55,10 +84,17 @@
/// If the last token read was an ERROR token, provide more info
/// or the empty string.
- string DescribeLastError();
+ std::string DescribeLastError();
- /// Start parsing some input.
- void Start(StringPiece filename, StringPiece input);
+ // XXX: Decide whether to use char* or size_t to represent lexer position.
+ // (We're using size_t for diagnostic reporting -- GetLastTokenOffset, but
+ // we're using char* for checking for the end-of-chunk.)
+ const char* GetPos() { return ofs_; }
+ void ResetPos(const char* pos) { ofs_ = pos; last_token_ = nullptr; }
+
+ size_t GetLastTokenOffset() {
+ return last_token_ ? last_token_ - input_.data() : 0;
+ }
/// Read a Token from the Token enum.
Token ReadToken();
@@ -66,40 +102,86 @@
/// Rewind to the last read Token.
void UnreadToken();
+ /// If the next token is a binding's indentation, this function reads it and
+ /// returns true. This function skips lines containing only a comment, which
+ /// are allowed in a block of indented bindings. (A completely blank line, on
+ /// the other hand, ends the block.)
+ bool PeekIndent();
+
/// If the next token is \a token, read it and return true.
bool PeekToken(Token token);
/// Read a simple identifier (a rule or variable name).
/// Returns false if a name can't be read.
- bool ReadIdent(string* out);
+ bool ReadIdent(StringPiece* out);
- /// Read a path (complete with $escapes).
- /// Returns false only on error, returned path may be empty if a delimiter
- /// (space, newline) is hit.
- bool ReadPath(EvalString* path, string* err) {
- return ReadEvalString(path, true, err);
- }
+ /// Parse the value in a "var = value" let binding and return the unprocessed
+ /// span of the value (escapes and variable expansions are left as-is). The
+ /// binding must end with a newline, which is consumed.
+ ///
+ /// The returned StringPiece includes the EOL terminator, "\r\n" or "\n".
+ /// Copying the StringPiece copies the terminator.
+ bool ReadBindingValue(StringPiece* out, std::string* err);
- /// Read the value side of a var = value line (complete with $escapes).
- /// Returns false only on error.
- bool ReadVarValue(EvalString* value, string* err) {
- return ReadEvalString(value, false, err);
- }
+ /// Try to parse a path from the manifest that's already canonical and doesn't
+ /// need evaluation. Returns the string if successful and return a 0-size
+ /// string otherwise. This function is a performance optimization.
+ StringPiece PeekCanonicalPath();
+
+ /// Read a ninja path (e.g. build/include declarations) and output a LexedPath
+ /// object with the position of the path in the loaded file's buffer. If there
+ /// is no path before the lexer sees a delimiter (e.g. space, newline, colon),
+ /// it returns true and outputs an empty string.
+ ///
+ /// On error, the function returns false and writes an error to *err.
+ ///
+ /// The delimiter is not included in the path's StringPiece, and (except for
+ /// space characters), the delimiter is left in the lexer's input stream.
+ ///
+ /// Because path evaluation requires the delimiter's presence, the path should
+ /// be evaluated before its underlying memory is freed, and the StringPiece
+ /// should not be duplicated. (e.g. Don't copy it into an std::string first.)
+ bool ReadPath(LexedPath* out, std::string* err);
/// Construct an error message with context.
- bool Error(const string& message, string* err);
+ bool Error(const std::string& message, std::string* err);
private:
+ /// Return the end of the file input (which points at a NUL byte).
+ const char* EndOfFile() { return input_.data() + input_.size(); }
+
+ /// Report an error on a NUL character, which could indicate EOF, but could
+ /// also be a stray NUL character in the file's content.
+ bool UnexpectedNulError(const char* pos, std::string* err);
+
/// Skip past whitespace (called after each read token/ident/etc.).
void EatWhitespace();
- /// Read a $-escaped string.
- bool ReadEvalString(EvalString* eval, bool path, string* err);
-
- StringPiece filename_;
+ std::string filename_;
StringPiece input_;
- const char* ofs_;
- const char* last_token_;
+ const char* ofs_ = nullptr;
+ const char* last_token_ = nullptr;
};
+void EvaluateBindingInScope(std::string* out_append,
+ StringPiece value,
+ ScopePosition pos);
+
+bool EvaluateBindingOnRule(std::string* out_append,
+ StringPiece value,
+ EdgeEval* edge_eval,
+ std::string* err);
+
+/// Used for lexer tests.
+std::string EvaluateBindingForTesting(StringPiece value);
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos);
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge);
+
+/// Used for lexer tests.
+std::string EvaluatePathForTesting(const LexedPath& path);
+
#endif // NINJA_LEXER_H_
diff --git a/src/lexer.in.cc b/src/lexer.in.cc
index c1fb822..63fa937 100644
--- a/src/lexer.in.cc
+++ b/src/lexer.in.cc
@@ -17,24 +17,79 @@
#include <stdio.h>
#include "eval_env.h"
+#include "graph.h"
#include "util.h"
-bool Lexer::Error(const string& message, string* err) {
+size_t AdvanceToNextManifestChunk(StringPiece content, size_t idx) {
+ assert(idx <= content.size());
+
+ // Iterate over each LF in the manifest, starting at the given index.
+ while (true) {
+ const void* next_line = memchr(content.data() + idx, '\n',
+ content.size() - idx);
+ if (next_line == nullptr) {
+ break;
+ }
+ idx = static_cast<const char*>(next_line) - content.data();
+ ++idx; // step over the LF
+
+ // The line must not be preceded by a line continuator. This logic can
+ // filter out more split candidates than strictly necessary:
+ // - The preceding line could have a comment that ends with a "$": "# $\n"
+ // - The preceding line could end with an escaped-dollar: "X=$$\n"
+ if ((idx >= 2 && content.substr(idx - 2, 2) == "$\n") ||
+ (idx >= 3 && content.substr(idx - 3, 3) == "$\r\n")) {
+ continue;
+ }
+
+ // Skip an indented line or a comment line, either of which could be part of
+ // an earlier declaration. Ninja allows unindented comments (as well as
+ // indented comments) inside a binding block, e.g.:
+ //
+ // build foo: cc
+ // # comment-line
+ // pool = link_pool
+ //
+ // Ninja doesn't allow blank lines in a binding block. This code could
+ // probably allow a chunk to start with a blank line, but it seems better if
+ // it doesn't.
+ if (idx >= content.size() ||
+ content[idx] == ' ' || content[idx] == '#' ||
+ content[idx] == '\r' || content[idx] == '\n') {
+ continue;
+ }
+
+ return idx;
+ }
+
+ return content.size();
+}
+
+bool DecorateErrorWithLocation(const std::string& filename,
+ const char* file_start,
+ size_t file_offset,
+ const std::string& message,
+ std::string* err) {
+ // Make a copy in case message and err alias.
+ std::string message_tmp = message;
+
// Compute line/column.
int line = 1;
- const char* line_start = input_.str_;
- for (const char* p = input_.str_; p < last_token_; ++p) {
+ const char* line_start = file_start;
+ const char* file_pos = file_start + file_offset;
+
+ for (const char* p = line_start; p < file_pos; ++p) {
if (*p == '\n') {
++line;
line_start = p + 1;
}
}
- int col = last_token_ ? (int)(last_token_ - line_start) : 0;
+ int col = (int)(file_pos - line_start);
char buf[1024];
- snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
+ snprintf(buf, sizeof(buf), "%s:%d: ", filename.c_str(), line);
*err = buf;
- *err += message + "\n";
+ *err += message_tmp + "\n";
// Add some context to the message.
const int kTruncateColumn = 72;
@@ -58,15 +113,16 @@
return false;
}
-Lexer::Lexer(const char* input) {
- Start("input", input);
+bool Lexer::Error(const std::string& message, std::string* err) {
+ return DecorateErrorWithLocation(filename_, input_.data(),
+ GetLastTokenOffset(), message, err);
}
-void Lexer::Start(StringPiece filename, StringPiece input) {
- filename_ = filename;
- input_ = input;
- ofs_ = input_.str_;
- last_token_ = NULL;
+bool Lexer::UnexpectedNulError(const char* pos, std::string* err) {
+ assert(*pos == '\0');
+ const char* msg = (pos == EndOfFile()) ? "unexpected EOF"
+ : "unexpected NUL byte";
+ return Error(msg, err);
}
const char* Lexer::TokenName(Token t) {
@@ -82,9 +138,11 @@
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case PIPEAT: return "'|@'";
case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
+ case TNUL: return "nul byte";
case TEOF: return "eof";
}
return NULL; // not reached
@@ -116,6 +174,7 @@
Lexer::Token Lexer::ReadToken() {
const char* p = ofs_;
const char* q;
+ const char* r;
const char* start;
Lexer::Token token;
for (;;) {
@@ -124,15 +183,17 @@
re2c:define:YYCTYPE = "unsigned char";
re2c:define:YYCURSOR = p;
re2c:define:YYMARKER = q;
+ re2c:define:YYCTXMARKER = r;
re2c:yyfill:enable = 0;
nul = "\000";
simple_varname = [a-zA-Z0-9_-]+;
varname = [a-zA-Z0-9_.-]+;
+ eol = "\n" | "\r\n";
+ comment = "#"[^\000\r\n]*;
- [ ]*"#"[^\000\n]*"\n" { continue; }
- [ ]*"\r\n" { token = NEWLINE; break; }
- [ ]*"\n" { token = NEWLINE; break; }
+ [ ]*comment / eol { continue; }
+ [ ]*eol { token = NEWLINE; break; }
[ ]+ { token = INDENT; break; }
"build" { token = BUILD; break; }
"pool" { token = POOL; break; }
@@ -140,12 +201,13 @@
"default" { token = DEFAULT; break; }
"=" { token = EQUALS; break; }
":" { token = COLON; break; }
+ "|@" { token = PIPEAT; break; }
"||" { token = PIPE2; break; }
"|" { token = PIPE; break; }
"include" { token = INCLUDE; break; }
"subninja" { token = SUBNINJA; break; }
varname { token = IDENT; break; }
- nul { token = TEOF; break; }
+ nul { token = (start == EndOfFile()) ? TEOF : TNUL; break; }
[^] { token = ERROR; break; }
*/
}
@@ -157,6 +219,21 @@
return token;
}
+bool Lexer::PeekIndent() {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+ /*!re2c
+ [ ]*comment eol { continue; }
+ [ ]*eol { last_token_ = ofs_ = start; return false; }
+ [ ]+ { last_token_ = start; ofs_ = p; return true; }
+ [^] { last_token_ = ofs_ = start; return false; }
+ */
+ }
+}
+
bool Lexer::PeekToken(Token token) {
Token t = ReadToken();
if (t == token)
@@ -172,22 +249,21 @@
ofs_ = p;
/*!re2c
[ ]+ { continue; }
- "$\r\n" { continue; }
- "$\n" { continue; }
+ "$" eol { continue; }
nul { break; }
[^] { break; }
*/
}
}
-bool Lexer::ReadIdent(string* out) {
+bool Lexer::ReadIdent(StringPiece* out) {
const char* p = ofs_;
const char* start;
for (;;) {
start = p;
/*!re2c
varname {
- out->assign(start, p - start);
+ *out = StringPiece(start, p - start);
break;
}
[^] {
@@ -202,66 +278,30 @@
return true;
}
-bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
+bool Lexer::ReadBindingValue(StringPiece* out, string* err) {
const char* p = ofs_;
const char* q;
const char* start;
for (;;) {
start = p;
/*!re2c
- [^$ :\r\n|\000]+ {
- eval->AddText(StringPiece(start, p - start));
+ ( [^$\r\n\000]
+ | "$" [$: ]
+ | "$" eol
+ | "${" varname "}"
+ | "$" simple_varname )+ {
continue;
}
- "\r\n" {
- if (path)
- p = start;
+ eol {
break;
}
- [ :|\n] {
- if (path) {
- p = start;
- break;
- } else {
- if (*start == '\n')
- break;
- eval->AddText(StringPiece(start, 1));
- continue;
- }
- }
- "$$" {
- eval->AddText(StringPiece("$", 1));
- continue;
- }
- "$ " {
- eval->AddText(StringPiece(" ", 1));
- continue;
- }
- "$\r\n"[ ]* {
- continue;
- }
- "$\n"[ ]* {
- continue;
- }
- "${"varname"}" {
- eval->AddSpecial(StringPiece(start + 2, p - start - 3));
- continue;
- }
- "$"simple_varname {
- eval->AddSpecial(StringPiece(start + 1, p - start - 1));
- continue;
- }
- "$:" {
- eval->AddText(StringPiece(":", 1));
- continue;
- }
"$". {
last_token_ = start;
return Error("bad $-escape (literal $ must be written as $$)", err);
}
nul {
last_token_ = start;
- return Error("unexpected EOF", err);
+ return UnexpectedNulError(start, err);
}
[^] {
last_token_ = start;
@@ -269,10 +309,192 @@
}
*/
}
+ *out = StringPiece(ofs_, p - ofs_);
last_token_ = start;
ofs_ = p;
- if (path)
- EatWhitespace();
// Non-path strings end in newlines, so there's no whitespace to eat.
return true;
}
+
+StringPiece Lexer::PeekCanonicalPath() {
+ auto finish = [this](const char* start, const char* end) {
+ ofs_ = end;
+ EatWhitespace();
+ return StringPiece(start, end - start);
+ };
+
+ const char* p = ofs_;
+ const char* q;
+ const char* r;
+ last_token_ = ofs_;
+
+ do {
+ /*!re2c
+ canon_no_dot = [^$ :|/.\r\n\000];
+ canon_any = canon_no_dot | ".";
+ canon_piece = ( canon_no_dot | "." canon_no_dot | ".." canon_any ) canon_any*;
+ canon_pieces = canon_piece ( "/" canon_piece )*;
+
+ // The Chromium gn manifests have many paths that start with "./" but are
+ // otherwise canonical. Allow them and strip off the leading "./".
+ ( "/" | "../"* ) canon_pieces / ([ :|] | eol) { return finish(ofs_, p); }
+ "./" "../"* canon_pieces / ([ :|] | eol) { return finish(ofs_ + 2, p); }
+ [^] { break; }
+ */
+ } while (false);
+
+ return {};
+}
+
+bool Lexer::ReadPath(LexedPath* out, std::string* err) {
+ const char* p = ofs_;
+ const char* q;
+ const char* start;
+ for (;;) {
+ start = p;
+ /*!re2c
+ ( [^$ :|\r\n\000]
+ | "$" [$: ]
+ | "$" eol [ ]*
+ | "${" varname "}"
+ | "$" simple_varname )+ {
+ continue;
+ }
+ [ :|] | eol {
+ p = start;
+ break;
+ }
+ "$". {
+ last_token_ = start;
+ return Error("bad $-escape (literal $ must be written as $$)", err);
+ }
+ nul {
+ last_token_ = start;
+ return UnexpectedNulError(start, err);
+ }
+ [^] {
+ last_token_ = start;
+ return Error(DescribeLastError(), err);
+ }
+ */
+ }
+ *out = {};
+ out->str_ = StringPiece(ofs_, p - ofs_);
+ last_token_ = start;
+ ofs_ = p;
+ EatWhitespace();
+ return true;
+}
+
+/// Append the let binding's evaluated value to the output string. The input
+/// StringPiece must include a valid binding terminator.
+template <typename EvalVar>
+static inline void EvaluateBinding(std::string* out_append, StringPiece value,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = value.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+ /*!re2c
+ [^$\r\n\000]+ { out_append->append(start, p - start); continue; }
+ "$" [$: ] { out_append->push_back(start[1]); continue; }
+ "$" eol [ ]* { continue; }
+ "${" varname "}" { expand(start + 2, p - 1); continue; }
+ "$" simple_varname { expand(start + 1, p); continue; }
+ eol { break; }
+ [^] { assert(false && "bad input in EvaluateBinding"); abort(); }
+ */
+ }
+ assert((p == value.data() + value.size()) &&
+ "bad end pos in EvaluateBinding");
+}
+
+void EvaluateBindingInScope(std::string* out_append, StringPiece value,
+ ScopePosition pos) {
+ EvaluateBinding(out_append, value,
+ [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+bool EvaluateBindingOnRule(std::string* out_append, StringPiece value,
+ EdgeEval* edge_eval, std::string* err) {
+ bool result = true;
+ EvaluateBinding(out_append, value,
+ [out_append, edge_eval, &result, err](const HashedStrView& var) {
+ result = result && edge_eval->EvaluateVariable(out_append, var, err);
+ });
+ return result;
+}
+
+std::string EvaluateBindingForTesting(StringPiece value) {
+ std::string result;
+ EvaluateBinding(&result, value, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
+
+/// Append an evaluated path to the output string.
+///
+/// This function does not canonicalize the output. Ninja canonicalizes paths for
+/// build nodes, but not all paths (e.g. It doesn't canonicalize paths to
+/// included ninja files.)
+template <typename EvalVar>
+static inline void EvaluatePath(std::string* out_append, const LexedPath& path,
+ EvalVar&& eval_var) {
+ auto expand = [&eval_var](const char* start, const char* end) {
+ StringPiece var(start, end - start);
+ eval_var(var);
+ };
+
+ const char* p = path.str_.data();
+ const char* q;
+
+ for (;;) {
+ const char* start = p;
+ /*!re2c
+ [^$ :|\r\n\000]+ { out_append->append(start, p - start); continue; }
+ "$" [$: ] { out_append->push_back(start[1]); continue; }
+ "$" eol [ ]* { continue; }
+ "${" varname "}" { expand(start + 2, p - 1); continue; }
+ "$" simple_varname { expand(start + 1, p); continue; }
+ [ :|] | eol { p = start; break; }
+ [^] { assert(false && "bad input in EvaluatePath"); abort(); }
+ */
+ }
+ assert((p == path.str_.data() + path.str_.size()) &&
+ "bad end pos in EvaluatePath");
+}
+
+void EvaluatePathInScope(std::string* out_append, const LexedPath& path,
+ ScopePosition pos) {
+ EvaluatePath(out_append, path, [out_append, &pos](const HashedStrView& var) {
+ Scope::EvaluateVariableAtPos(out_append, var, pos);
+ });
+}
+
+void EvaluatePathOnEdge(std::string* out_append, const LexedPath& path,
+ const Edge& edge) {
+ EvaluatePath(out_append, path, [out_append, &edge](const HashedStrView& var) {
+ // First look for a binding on the edge itself, then fall back to the
+ // edge's enclosing scope.
+ if (edge.EvaluateVariableSelfOnly(out_append, var))
+ return;
+ Scope::EvaluateVariableAtPos(out_append, var, edge.pos_.scope_pos());
+ });
+}
+
+std::string EvaluatePathForTesting(const LexedPath& path) {
+ std::string result;
+ EvaluatePath(&result, path, [&result](StringPiece var) {
+ result += "[$" + var.AsString() + "]";
+ });
+ return result;
+}
diff --git a/src/lexer_test.cc b/src/lexer_test.cc
index 331d8e1..1b69bed 100644
--- a/src/lexer_test.cc
+++ b/src/lexer_test.cc
@@ -18,28 +18,30 @@
#include "test.h"
TEST(Lexer, ReadVarValue) {
- Lexer lexer("plain text $var $VaR ${x}\n");
- EvalString eval;
- string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ Lexer lexer("plain text $var $VaR ${x}\nmore text");
+ StringPiece uneval;
+ std::string err;
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[plain text ][$var][ ][$VaR][ ][$x]",
- eval.Serialize());
+ EXPECT_EQ("plain text $var $VaR ${x}\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ("plain text [$var] [$VaR] [$x]", eval);
}
TEST(Lexer, ReadEvalStringEscapes) {
- Lexer lexer("$ $$ab c$: $\ncde\n");
- EvalString eval;
- string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ Lexer lexer("$ $$ab c$: $\ncde\nmore text");
+ StringPiece uneval;
+ std::string err;
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[ $ab c: cde]",
- eval.Serialize());
+ EXPECT_EQ("$ $$ab c$: $\ncde\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ(" $ab c: cde", eval);
}
TEST(Lexer, ReadIdent) {
Lexer lexer("foo baR baz_123 foo-bar");
- string ident;
+ StringPiece ident;
EXPECT_TRUE(lexer.ReadIdent(&ident));
EXPECT_EQ("foo", ident);
EXPECT_TRUE(lexer.ReadIdent(&ident));
@@ -54,23 +56,71 @@
// Verify that ReadIdent includes dots in the name,
// but in an expansion $bar.dots stops at the dot.
Lexer lexer("foo.dots $bar.dots ${bar.dots}\n");
- string ident;
+ StringPiece ident;
EXPECT_TRUE(lexer.ReadIdent(&ident));
EXPECT_EQ("foo.dots", ident);
- EvalString eval;
+ StringPiece uneval;
string err;
- EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+ EXPECT_TRUE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("", err);
- EXPECT_EQ("[$bar][.dots ][$bar.dots]",
- eval.Serialize());
+ EXPECT_EQ("$bar.dots ${bar.dots}\n", uneval.AsString());
+ std::string eval = EvaluateBindingForTesting(uneval);
+ EXPECT_EQ("[$bar].dots [$bar.dots]", eval);
+}
+
+TEST(Lexer, PeekCanonicalPath) {
+ auto peek_canon = [](const char* pattern) {
+ Lexer lexer(pattern);
+ return lexer.PeekCanonicalPath().AsString();
+ };
+ EXPECT_EQ("", peek_canon(""));
+ EXPECT_EQ("", peek_canon(" : "));
+ EXPECT_EQ("", peek_canon("/ : "));
+ EXPECT_EQ("", peek_canon("/foo/ : "));
+ EXPECT_EQ("", peek_canon("/foo/../bar : "));
+ EXPECT_EQ("", peek_canon("/../foo/bar : "));
+
+ EXPECT_EQ("", peek_canon("foo$$bar : "));
+ EXPECT_EQ("", peek_canon("foo${bar}baz : "));
+
+ EXPECT_EQ("/foo", peek_canon("/foo : "));
+ EXPECT_EQ("/foo/bar", peek_canon("/foo/bar : "));
+ EXPECT_EQ("foo", peek_canon("foo : "));
+ EXPECT_EQ("foo/bar", peek_canon("foo/bar : "));
+ EXPECT_EQ("../foo/bar", peek_canon("../foo/bar : "));
+ EXPECT_EQ("../../foo/bar", peek_canon("../../foo/bar : "));
+
+ EXPECT_EQ("foo", peek_canon("./foo : "));
+ EXPECT_EQ("../../foo/bar", peek_canon("./../../foo/bar : "));
+}
+
+TEST(Lexer, ReadPath) {
+ Lexer lexer("x$$y$ z$:: $bar.dots\n");
+ std::string err;
+ LexedPath uneval;
+ std::string eval;
+
+ EXPECT_TRUE(lexer.ReadPath(&uneval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("x$$y$ z$:", uneval.str_.AsString());
+ eval = EvaluatePathForTesting(uneval);
+ EXPECT_EQ("x$y z:", eval);
+
+ EXPECT_EQ(Lexer::COLON, lexer.ReadToken());
+
+ EXPECT_TRUE(lexer.ReadPath(&uneval, &err));
+ EXPECT_EQ("", err);
+ EXPECT_EQ("$bar.dots", uneval.str_.AsString());
+ eval = EvaluatePathForTesting(uneval);
+ EXPECT_EQ("[$bar].dots", eval);
}
TEST(Lexer, Error) {
Lexer lexer("foo$\nbad $");
- EvalString eval;
+ StringPiece uneval;
string err;
- ASSERT_FALSE(lexer.ReadVarValue(&eval, &err));
+ ASSERT_FALSE(lexer.ReadBindingValue(&uneval, &err));
EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
"bad $\n"
" ^ near here"
@@ -94,3 +144,40 @@
EXPECT_EQ(Lexer::ERROR, token);
EXPECT_EQ("tabs are not allowed, use spaces", lexer.DescribeLastError());
}
+
+TEST(Lexer, AdvanceToNextManifestChunk) {
+ auto advance = [](std::string input) -> std::string {
+ // The input string may optionally have a '^' marking where the lexer should
+ // start scanning for a chunk.
+ size_t idx = input.find_first_of('^');
+ if (idx == std::string::npos) {
+ idx = 0;
+ } else {
+ input = input.substr(0, idx) + input.substr(idx + 1);
+ }
+ idx = AdvanceToNextManifestChunk(input, idx);
+ // The return string has a '^' marking where the manifest was split.
+ return input.substr(0, idx) + "^" + input.substr(idx);
+ };
+
+ EXPECT_EQ("^", advance(""));
+ EXPECT_EQ("A\n^B\n", advance("A\nB\n"));
+ EXPECT_EQ("\n^B\n", advance("\nB\n"));
+ EXPECT_EQ("\0\0\0^"_s, advance("\0\0\0"_s));
+
+ // An LF preceded by "$" or "$\r" might be part of a line continuator, so we
+ // skip it when looking for a place to split the manifest.
+ EXPECT_EQ("A$\nB\n^C", advance("^A$\nB\nC"));
+ EXPECT_EQ("A$\nB\n^C", advance("A^$\nB\nC"));
+ EXPECT_EQ("A$\nB\n^C", advance("A$^\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("^A$\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A^$\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A$^\r\nB\nC"));
+ EXPECT_EQ("A$\r\nB\n^C", advance("A$\r^\nB\nC"));
+
+ // Skip over blank lines and comment lines.
+ EXPECT_EQ("\n^", advance("\n"));
+ EXPECT_EQ("\n\n^A\n", advance("\n\nA\n"));
+ EXPECT_EQ("\n \n^A", advance("\n \nA"));
+ EXPECT_EQ("\n#\n^A", advance("\n#\nA"));
+}
diff --git a/src/manifest_chunk_parser.cc b/src/manifest_chunk_parser.cc
new file mode 100644
index 0000000..ff36752
--- /dev/null
+++ b/src/manifest_chunk_parser.cc
@@ -0,0 +1,433 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "manifest_chunk_parser.h"
+
+#include <assert.h>
+
+#include <thread>
+#include <unordered_map>
+
+#include "lexer.h"
+#include "metrics.h"
+#include "thread_pool.h"
+#include "util.h"
+
+namespace manifest_chunk {
+
+class ChunkParser {
+ const LoadedFile& file_;
+ Lexer lexer_;
+ const char* chunk_end_ = nullptr;
+ std::vector<ParserItem>* out_ = nullptr;
+ Clump* current_clump_ = nullptr;
+
+ Clump* MakeClump() {
+ if (current_clump_)
+ return current_clump_;
+ current_clump_ = new Clump { file_ };
+ out_->push_back(current_clump_);
+ return current_clump_;
+ }
+
+ void OutItem(ParserItem item) {
+ current_clump_ = nullptr;
+ out_->push_back(item);
+ }
+
+ bool OutError(const std::string& err) {
+ OutItem(new Error { err });
+ return false;
+ }
+
+ bool LexerError(const std::string& message);
+ bool ExpectToken(Lexer::Token expected);
+ bool ParseLet(StringPiece* key, StringPiece* value);
+ bool ParseFileInclude(bool new_scope);
+ bool ParsePool();
+ bool ParseDefault();
+ bool ParseBinding();
+ bool ParseRule();
+ bool ParseEdge();
+
+public:
+ ChunkParser(const LoadedFile& file,
+ StringPiece chunk_content,
+ std::vector<ParserItem>* out)
+ : file_(file),
+ lexer_(file.filename(), file.content(), chunk_content.data()),
+ chunk_end_(chunk_content.data() + chunk_content.size()),
+ out_(out) {}
+
+ bool ParseChunk();
+};
+
+bool ChunkParser::LexerError(const std::string& message) {
+ std::string err;
+ lexer_.Error(message, &err);
+ return OutError(err);
+}
+
+bool ChunkParser::ExpectToken(Lexer::Token expected) {
+ Lexer::Token token = lexer_.ReadToken();
+ if (token != expected) {
+ std::string message = string("expected ") + Lexer::TokenName(expected);
+ message += string(", got ") + Lexer::TokenName(token);
+ message += Lexer::TokenErrorHint(expected);
+ return LexerError(message);
+ }
+ return true;
+}
+
+bool ChunkParser::ParseLet(StringPiece* key, StringPiece* value) {
+ if (!lexer_.ReadIdent(key))
+ return LexerError("expected variable name");
+ if (!ExpectToken(Lexer::EQUALS))
+ return false;
+ std::string err;
+ if (!lexer_.ReadBindingValue(value, &err))
+ return OutError(err);
+ return true;
+}
+
+bool ChunkParser::ParseFileInclude(bool new_scope) {
+ Include* include = new Include();
+ include->new_scope_ = new_scope;
+ std::string err;
+ if (!lexer_.ReadPath(&include->path_, &err))
+ return OutError(err);
+ include->diag_pos_ = lexer_.GetLastTokenOffset();
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+ OutItem(include);
+ return true;
+}
+
+bool ChunkParser::ParsePool() {
+ Pool* pool = new Pool();
+
+ StringPiece name;
+ if (!lexer_.ReadIdent(&name))
+ return LexerError("expected pool name");
+ pool->name_ = name;
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ pool->parse_state_.pool_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece value;
+ if (!ParseLet(&key, &value))
+ return false;
+
+ if (key == "depth") {
+ pool->parse_state_.depth = value;
+ pool->parse_state_.depth_diag_pos = lexer_.GetLastTokenOffset();
+ } else {
+ return LexerError("unexpected variable '" + key.AsString() + "'");
+ }
+ }
+
+ if (pool->parse_state_.depth.empty())
+ return LexerError("expected 'depth =' line");
+
+ Clump* clump = MakeClump();
+ pool->pos_ = clump->AllocNextPos();
+ clump->pools_.push_back(pool);
+ return true;
+}
+
+bool ChunkParser::ParseDefault() {
+ bool first = true;
+ while (true) {
+ LexedPath path;
+ std::string err;
+ if (!lexer_.ReadPath(&path, &err))
+ return OutError(err);
+ if (path.str_.empty()) {
+ if (first)
+ return LexerError("expected target name");
+ else
+ break;
+ }
+ first = false;
+
+ DefaultTarget* target = new DefaultTarget();
+ target->parsed_path_ = std::move(path);
+ target->diag_pos_ = lexer_.GetLastTokenOffset();
+
+ Clump* clump = MakeClump();
+ target->pos_ = clump->AllocNextPos();
+ clump->default_targets_.push_back(target);
+ }
+ return ExpectToken(Lexer::NEWLINE);
+}
+
+static const HashedStrView kNinjaRequiredVersion { "ninja_required_version" };
+
+bool ChunkParser::ParseBinding() {
+ Binding* binding = new Binding();
+
+ lexer_.UnreadToken();
+ StringPiece name;
+ if (!ParseLet(&name, &binding->parsed_value_))
+ return false;
+ binding->name_ = name;
+
+ // Record a ninja_required_version binding specially. We want to do this
+ // version check ASAP in case we encounter unexpected syntax. We can't do the
+ // check immediately because the version string is evaluated and might need
+ // bindings from an earlier chunk. We could probably do this version check as
+ // part of ordinary scope setup while processing a Clump, but keeping it
+ // separate guarantees that the version check is done early enough.
+ if (binding->name_ == kNinjaRequiredVersion)
+ OutItem(new RequiredVersion { binding->parsed_value_ });
+
+ Clump* clump = MakeClump();
+ binding->pos_ = clump->AllocNextPos();
+ clump->bindings_.push_back(binding);
+ return true;
+}
+
+static const HashedStrView kRspFile { "rspfile" };
+static const HashedStrView kRspFileContent { "rspfile_content" };
+static const HashedStrView kCommand { "command" };
+
+bool ChunkParser::ParseRule() {
+ Rule* rule = new Rule();
+
+ StringPiece rule_name;
+ if (!lexer_.ReadIdent(&rule_name))
+ return LexerError("expected rule name");
+ rule->name_ = rule_name;
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ rule->parse_state_.rule_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece value;
+ if (!ParseLet(&key, &value))
+ return false;
+
+ if (!Rule::IsReservedBinding(key)) {
+ // Die on other keyvals for now; revisit if we want to add a scope here.
+ // If we allow arbitrary key values here, we'll need to revisit how cycle
+ // detection works when evaluating a rule variable.
+ return LexerError("unexpected variable '" + key.AsString() + "'");
+ }
+
+ rule->bindings_.emplace_back(std::piecewise_construct,
+ std::tuple<StringPiece>(key),
+ std::tuple<const char*, size_t>(value.data(),
+ value.size()));
+ }
+
+ if (static_cast<bool>(rule->GetBinding(kRspFile)) !=
+ static_cast<bool>(rule->GetBinding(kRspFileContent))) {
+ return LexerError("rspfile and rspfile_content need to be both specified");
+ }
+
+ if (rule->GetBinding(kCommand) == nullptr)
+ return LexerError("expected 'command =' line");
+
+ Clump* clump = MakeClump();
+ rule->pos_ = clump->AllocNextPos();
+ clump->rules_.push_back(rule);
+ return true;
+}
+
+bool ChunkParser::ParseEdge() {
+ Edge* edge = new Edge();
+
+ auto parse_path_list = [this, edge](Edge::DeferredPathList::Type type, int& count) -> bool {
+ const char* start_pos = lexer_.GetPos();
+ while (true) {
+ LexedPath path;
+ std::string err;
+ if (!lexer_.ReadPath(&path, &err))
+ return OutError(err);
+ if (path.str_.empty()) {
+ // In the initial parsing pass, the manifest's bindings aren't ready
+ // yet, so paths can't be evaluated. Rather than store the path itself
+ // (wasting memory), store just enough information to parse the path
+ // lists again once bindings are ready.
+ edge->parse_state_.deferred_path_lists.push_back({
+ start_pos, type, count,
+ });
+ return true;
+ }
+ count++;
+ }
+ };
+
+ if (!parse_path_list(Edge::DeferredPathList::OUTPUT,
+ edge->explicit_outs_))
+ return false;
+
+ // Add all implicit outs, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE)) {
+ if (!parse_path_list(Edge::DeferredPathList::OUTPUT,
+ edge->implicit_outs_))
+ return false;
+ }
+
+ if (edge->explicit_outs_ + edge->implicit_outs_ == 0)
+ return LexerError("expected path");
+
+ if (!ExpectToken(Lexer::COLON))
+ return false;
+
+ StringPiece rule_name;
+ if (!lexer_.ReadIdent(&rule_name))
+ return LexerError("expected build command name");
+ edge->parse_state_.rule_name = rule_name;
+ edge->parse_state_.rule_name_diag_pos = lexer_.GetLastTokenOffset();
+
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->explicit_deps_))
+ return false;
+
+ // Add all implicit deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE)) {
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->implicit_deps_))
+ return false;
+ }
+
+ // Add all order-only deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPE2)) {
+ if (!parse_path_list(Edge::DeferredPathList::INPUT,
+ edge->order_only_deps_))
+ return false;
+ }
+
+ // Add all validation deps, counting how many as we go.
+ if (lexer_.PeekToken(Lexer::PIPEAT)) {
+ if (!parse_path_list(Edge::DeferredPathList::VALIDATION,
+ edge->validation_deps_))
+ return false;
+ }
+
+
+ if (!ExpectToken(Lexer::NEWLINE))
+ return false;
+
+ while (lexer_.PeekIndent()) {
+ StringPiece key;
+ StringPiece val;
+ if (!ParseLet(&key, &val))
+ return false;
+
+ std::tuple<const char*, size_t> val_ctor_params(val.data(), val.size());
+ edge->unevaled_bindings_.emplace_back(std::piecewise_construct,
+ std::tuple<StringPiece>(key),
+ val_ctor_params);
+ }
+
+ edge->parse_state_.final_diag_pos = lexer_.GetLastTokenOffset();
+
+ Clump* clump = MakeClump();
+ edge->pos_ = clump->AllocNextPos();
+ clump->edges_.push_back(edge);
+ clump->edge_output_count_ += edge->explicit_outs_;
+ return true;
+}
+
+bool ChunkParser::ParseChunk() {
+ while (true) {
+ if (lexer_.GetPos() >= chunk_end_) {
+ assert(lexer_.GetPos() == chunk_end_ &&
+ "lexer scanned beyond the end of a manifest chunk");
+ return true;
+ }
+
+ Lexer::Token token = lexer_.ReadToken();
+ bool success = true;
+ switch (token) {
+ case Lexer::INCLUDE: success = ParseFileInclude(false); break;
+ case Lexer::SUBNINJA: success = ParseFileInclude(true); break;
+ case Lexer::POOL: success = ParsePool(); break;
+ case Lexer::DEFAULT: success = ParseDefault(); break;
+ case Lexer::IDENT: success = ParseBinding(); break;
+ case Lexer::RULE: success = ParseRule(); break;
+ case Lexer::BUILD: success = ParseEdge(); break;
+ case Lexer::NEWLINE: break;
+ case Lexer::ERROR: return LexerError(lexer_.DescribeLastError());
+ case Lexer::TNUL: return LexerError("unexpected NUL byte");
+ case Lexer::TEOF:
+ assert(false && "EOF should have been detected before reading a token");
+ break;
+ default:
+ return LexerError(std::string("unexpected ") + Lexer::TokenName(token));
+ }
+ if (!success) return false;
+ }
+ return false; // not reached
+}
+
+void ParseChunk(const LoadedFile& file, StringPiece chunk_content,
+ std::vector<ParserItem>* out) {
+ ChunkParser parser(file, chunk_content, out);
+ if (!parser.ParseChunk()) {
+ assert(!out->empty());
+ assert(out->back().kind == ParserItem::kError);
+ assert(!out->back().u.error->msg_.empty());
+ }
+}
+
+std::vector<StringPiece> SplitManifestIntoChunks(StringPiece input) {
+ METRIC_RECORD(".ninja load : split manifest");
+
+ // The lexer requires that a NUL character appear after the file's content.
+ // Make the NUL implicit for the rest of the manifest parsing.
+ //
+ // In general, parsing a manifest chunk examines the terminating text after
+ // the chunk's explicit end. For example, suppose we have two consecutive
+ // chunks:
+ // - "build foo: gen\n pool = mypool\n"
+ // - "build bar: gen\n"
+ // The parser for the "foo" chunk will examine the start of "build" from the
+ // "bar" chunk when it looks for another indented edge binding.
+ assert(!input.empty() && input.back() == '\0' &&
+ "input lacks a NUL terminator");
+ input.remove_suffix(1);
+
+ // For efficiency, avoid splitting small files. kChunkMinSize can be lowered
+ // to 0 for testing.
+ const size_t kChunkMinSize = 1024 * 1024;
+
+ const size_t chunk_count = GetOptimalThreadPoolJobCount();
+ const size_t chunk_size = std::max(kChunkMinSize,
+ input.size() / chunk_count + 1);
+
+ std::vector<StringPiece> result;
+ result.reserve(chunk_count);
+
+ size_t start = 0;
+ while (start < input.size()) {
+ size_t next = std::min(start + chunk_size, input.size());
+ next = AdvanceToNextManifestChunk(input, next);
+ result.push_back(input.substr(start, next - start));
+ start = next;
+ }
+
+ return result;
+}
+
+} // namespace manifest_chunk
diff --git a/src/manifest_chunk_parser.h b/src/manifest_chunk_parser.h
new file mode 100644
index 0000000..665a805
--- /dev/null
+++ b/src/manifest_chunk_parser.h
@@ -0,0 +1,112 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_MANIFEST_CHUNK_PARSER_H_
+#define NINJA_MANIFEST_CHUNK_PARSER_H_
+
+#include <stdlib.h>
+
+#include <string>
+#include <vector>
+
+#include "disk_interface.h"
+#include "eval_env.h"
+#include "graph.h"
+#include "state.h"
+#include "string_piece.h"
+
+struct Lexer;
+
+namespace manifest_chunk {
+
+struct Error {
+ std::string msg_;
+};
+
+struct RequiredVersion {
+ StringPiece version_;
+};
+
+struct Include {
+ LexedPath path_;
+ bool new_scope_ = false;
+ size_t diag_pos_ = 0;
+};
+
+struct DefaultTarget {
+ RelativePosition pos_;
+ LexedPath parsed_path_;
+ size_t diag_pos_ = 0;
+};
+
+/// A group of manifest declarations generated while parsing a chunk.
+struct Clump {
+ Clump(const LoadedFile& file) : file_(file) {}
+
+ const LoadedFile& file_;
+ BasePosition pos_;
+
+ std::vector<Binding*> bindings_;
+ std::vector<Rule*> rules_;
+ std::vector<Pool*> pools_;
+ std::vector<Edge*> edges_;
+ std::vector<DefaultTarget*> default_targets_;
+
+ /// A count of non-implicit outputs across all edges.
+ size_t edge_output_count_ = 0;
+
+ DeclIndex decl_count() const { return next_index_; }
+
+ /// Allocate an index within the clump. Once the parallelized chunk parsing is
+ /// finished, each clump's base position will be computed, giving every clump
+ /// item both a DFS "depth-first-search" position and a position within the
+ /// tree of scopes.
+ RelativePosition AllocNextPos() { return { &pos_, next_index_++ }; }
+
+private:
+ DeclIndex next_index_ = 0;
+};
+
+/// This class could be replaced with std::variant from C++17.
+struct ParserItem {
+ enum Kind {
+ kError, kRequiredVersion, kInclude, kClump
+ };
+
+ Kind kind;
+
+ union {
+ Error* error;
+ RequiredVersion* required_version;
+ Include* include;
+ Clump* clump;
+ } u;
+
+ ParserItem(Error* val) : kind(kError) { u.error = val; }
+ ParserItem(RequiredVersion* val) : kind(kRequiredVersion) { u.required_version = val; }
+ ParserItem(Include* val) : kind(kInclude) { u.include = val; }
+ ParserItem(Clump* val) : kind(kClump) { u.clump = val; }
+};
+
+/// Parse a chunk of a manifest. If the parser encounters a syntactic error, the
+/// final item will be an error object.
+void ParseChunk(const LoadedFile& file, StringPiece chunk_content,
+ std::vector<ParserItem>* out);
+
+/// Split the input into chunks of declarations.
+std::vector<StringPiece> SplitManifestIntoChunks(StringPiece input);
+
+} // namespace manifest_chunk
+
+#endif // NINJA_MANIFEST_CHUNK_PARSER_H_
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 27c423b..1e4246c 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -1,4 +1,4 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
+// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,434 +14,597 @@
#include "manifest_parser.h"
-#include <stdio.h>
-#include <stdlib.h>
+#include <assert.h>
+
+#include <unordered_map>
#include <vector>
#include "disk_interface.h"
-#include "graph.h"
+#include "eval_env.h"
+#include "lexer.h"
#include "metrics.h"
+#include "parallel_map.h"
#include "state.h"
#include "util.h"
#include "version.h"
+#include "manifest_chunk_parser.h"
-ManifestParser::ManifestParser(State* state, FileReader* file_reader,
- ManifestParserOptions options)
- : state_(state), file_reader_(file_reader),
- options_(options), quiet_(false) {
- env_ = &state->bindings_;
+using manifest_chunk::Clump;
+using manifest_chunk::DefaultTarget;
+using manifest_chunk::Include;
+using manifest_chunk::ParserItem;
+using manifest_chunk::RequiredVersion;
+
+namespace {
+
+static bool DecorateError(const LoadedFile& file, size_t file_offset,
+ const std::string& message, std::string* err) {
+ assert(file_offset <= file.content().size());
+ return DecorateErrorWithLocation(file.filename(), file.content().data(),
+ file_offset, message, err);
}
-bool ManifestParser::Load(const string& filename, string* err, Lexer* parent) {
- METRIC_RECORD(".ninja parse");
- string contents;
- string read_err;
- if (file_reader_->ReadFile(filename, &contents, &read_err) != FileReader::Okay) {
- *err = "loading '" + filename + "': " + read_err;
- if (parent)
- parent->Error(string(*err), err);
+/// Split a single manifest file into chunks, parse the chunks in parallel, and
+/// return the resulting parser output.
+static std::vector<ParserItem> ParseManifestChunks(const LoadedFile& file,
+ ThreadPool* thread_pool) {
+ std::vector<ParserItem> result;
+ const std::vector<StringPiece> chunk_views =
+ manifest_chunk::SplitManifestIntoChunks(file.content_with_nul());
+
+ METRIC_RECORD(".ninja load : parse chunks");
+
+ for (std::vector<ParserItem>& chunk_items :
+ ParallelMap(thread_pool, chunk_views, [&file](StringPiece view) {
+ std::vector<ParserItem> chunk_items;
+ manifest_chunk::ParseChunk(file, view, &chunk_items);
+ return chunk_items;
+ })) {
+ std::move(chunk_items.begin(), chunk_items.end(),
+ std::back_inserter(result));
+ }
+ return result;
+}
+
+static void ReserveSpaceInScopeTables(Scope* scope,
+ const std::vector<ParserItem>& items) {
+ // Reserve extra space in the scope's bindings/rules hash tables.
+ METRIC_RECORD(".ninja load : alloc scope tables");
+ size_t new_bindings = 0;
+ size_t new_rules = 0;
+ for (const auto& item : items) {
+ if (item.kind == ParserItem::kClump) {
+ Clump* clump = item.u.clump;
+ new_bindings += clump->bindings_.size();
+ new_rules += clump->rules_.size();
+ }
+ }
+ scope->ReserveTableSpace(new_bindings, new_rules);
+}
+
+struct ManifestFileSet {
+ ManifestFileSet(FileReader* file_reader) : file_reader_(file_reader) {}
+
+ bool LoadFile(const std::string& filename, const LoadedFile** result,
+ std::string* err);
+
+ ~ManifestFileSet() {
+ // It can take a surprising long time to unmap all the manifest files
+ // (e.g. ~70 ms for Android, 25 ms for Chromium).
+ METRIC_RECORD(".ninja load : unload files");
+ loaded_files_.clear();
+ }
+
+private:
+ FileReader *file_reader_ = nullptr;
+
+ /// Keep all the manifests in memory until all the manifest parsing work is
+ /// complete, because intermediate parsing results (i.e. StringPiece) may
+ /// point into the loaded files.
+ std::vector<std::unique_ptr<LoadedFile>> loaded_files_;
+};
+
+bool ManifestFileSet::LoadFile(const std::string& filename,
+ const LoadedFile** result,
+ std::string* err) {
+ std::unique_ptr<LoadedFile> loaded_file;
+ if (file_reader_->LoadFile(filename, &loaded_file, err) != FileReader::Okay) {
+ *err = "loading '" + filename + "': " + *err;
return false;
}
-
- // The lexer needs a nul byte at the end of its input, to know when it's done.
- // It takes a StringPiece, and StringPiece's string constructor uses
- // string::data(). data()'s return value isn't guaranteed to be
- // null-terminated (although in practice - libc++, libstdc++, msvc's stl --
- // it is, and C++11 demands that too), so add an explicit nul byte.
- contents.resize(contents.size() + 1);
-
- return Parse(filename, contents, err);
-}
-
-bool ManifestParser::Parse(const string& filename, const string& input,
- string* err) {
- lexer_.Start(filename, input);
-
- for (;;) {
- Lexer::Token token = lexer_.ReadToken();
- switch (token) {
- case Lexer::POOL:
- if (!ParsePool(err))
- return false;
- break;
- case Lexer::BUILD:
- if (!ParseEdge(err))
- return false;
- break;
- case Lexer::RULE:
- if (!ParseRule(err))
- return false;
- break;
- case Lexer::DEFAULT:
- if (!ParseDefault(err))
- return false;
- break;
- case Lexer::IDENT: {
- lexer_.UnreadToken();
- string name;
- EvalString let_value;
- if (!ParseLet(&name, &let_value, err))
- return false;
- string value = let_value.Evaluate(env_);
- // Check ninja_required_version immediately so we can exit
- // before encountering any syntactic surprises.
- if (name == "ninja_required_version")
- CheckNinjaVersion(value);
- env_->AddBinding(name, value);
- break;
- }
- case Lexer::INCLUDE:
- if (!ParseFileInclude(false, err))
- return false;
- break;
- case Lexer::SUBNINJA:
- if (!ParseFileInclude(true, err))
- return false;
- break;
- case Lexer::ERROR: {
- return lexer_.Error(lexer_.DescribeLastError(), err);
- }
- case Lexer::TEOF:
- return true;
- case Lexer::NEWLINE:
- break;
- default:
- return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
- err);
- }
- }
- return false; // not reached
-}
-
-
-bool ManifestParser::ParsePool(string* err) {
- string name;
- if (!lexer_.ReadIdent(&name))
- return lexer_.Error("expected pool name", err);
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
-
- if (state_->LookupPool(name) != NULL)
- return lexer_.Error("duplicate pool '" + name + "'", err);
-
- int depth = -1;
-
- while (lexer_.PeekToken(Lexer::INDENT)) {
- string key;
- EvalString value;
- if (!ParseLet(&key, &value, err))
- return false;
-
- if (key == "depth") {
- string depth_string = value.Evaluate(env_);
- depth = atol(depth_string.c_str());
- if (depth < 0)
- return lexer_.Error("invalid pool depth", err);
- } else {
- return lexer_.Error("unexpected variable '" + key + "'", err);
- }
- }
-
- if (depth < 0)
- return lexer_.Error("expected 'depth =' line", err);
-
- state_->AddPool(new Pool(name, depth));
+ loaded_files_.push_back(std::move(loaded_file));
+ *result = loaded_files_.back().get();
return true;
}
+struct DfsParser {
+ DfsParser(ManifestFileSet* file_set, State* state, ThreadPool* thread_pool)
+ : file_set_(file_set), state_(state), thread_pool_(thread_pool) {}
-bool ManifestParser::ParseRule(string* err) {
- string name;
- if (!lexer_.ReadIdent(&name))
- return lexer_.Error("expected rule name", err);
+private:
+ void HandleRequiredVersion(const RequiredVersion& item, Scope* scope);
+ bool HandleInclude(const Include& item, const LoadedFile& file, Scope* scope,
+ const LoadedFile** child_file, Scope** child_scope,
+ std::string* err);
+ bool HandlePool(Pool* pool, const LoadedFile& file, std::string* err);
+ bool HandleClump(Clump* clump, const LoadedFile& file, Scope* scope,
+ std::string* err);
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
+public:
+ /// Load the tree of manifest files and do initial parsing of all the chunks.
+ /// This function always runs on the main thread.
+ bool LoadManifestTree(const LoadedFile& file, Scope* scope,
+ std::vector<Clump*>* out_clumps, std::string* err);
- if (env_->LookupRuleCurrentScope(name) != NULL)
- return lexer_.Error("duplicate rule '" + name + "'", err);
+private:
+ ManifestFileSet* file_set_;
+ State* state_;
+ ThreadPool* thread_pool_;
+};
- Rule* rule = new Rule(name); // XXX scoped_ptr
+void DfsParser::HandleRequiredVersion(const RequiredVersion& item,
+ Scope* scope) {
+ std::string version;
+ EvaluateBindingInScope(&version, item.version_,
+ scope->GetCurrentEndOfScope());
+ CheckNinjaVersion(version);
+}
- while (lexer_.PeekToken(Lexer::INDENT)) {
- string key;
- EvalString value;
- if (!ParseLet(&key, &value, err))
- return false;
-
- if (Rule::IsReservedBinding(key)) {
- rule->AddBinding(key, value);
- } else {
- // Die on other keyvals for now; revisit if we want to add a
- // scope here.
- return lexer_.Error("unexpected variable '" + key + "'", err);
- }
+bool DfsParser::HandleInclude(const Include& include, const LoadedFile& file,
+ Scope* scope, const LoadedFile** child_file,
+ Scope** child_scope, std::string* err) {
+ std::string path;
+ EvaluatePathInScope(&path, include.path_, scope->GetCurrentEndOfScope());
+ if (!file_set_->LoadFile(path, child_file, err)) {
+ return DecorateError(file, include.diag_pos_, std::string(*err), err);
}
-
- if (rule->bindings_["rspfile"].empty() !=
- rule->bindings_["rspfile_content"].empty()) {
- return lexer_.Error("rspfile and rspfile_content need to be "
- "both specified", err);
+ if (include.new_scope_) {
+ *child_scope = new Scope(scope->GetCurrentEndOfScope());
+ } else {
+ *child_scope = scope;
}
-
- if (rule->bindings_["command"].empty())
- return lexer_.Error("expected 'command =' line", err);
-
- env_->AddRule(rule);
return true;
}
-bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
- if (!lexer_.ReadIdent(key))
- return lexer_.Error("expected variable name", err);
- if (!ExpectToken(Lexer::EQUALS, err))
- return false;
- if (!lexer_.ReadVarValue(value, err))
- return false;
+bool DfsParser::HandlePool(Pool* pool, const LoadedFile& file,
+ std::string* err) {
+ std::string depth_string;
+ EvaluateBindingInScope(&depth_string, pool->parse_state_.depth,
+ pool->pos_.scope_pos());
+ pool->depth_ = atol(depth_string.c_str());
+ if (pool->depth_ < 0) {
+ return DecorateError(file, pool->parse_state_.depth_diag_pos,
+ "invalid pool depth", err);
+ }
+ if (!state_->AddPool(pool)) {
+ return DecorateError(file, pool->parse_state_.pool_name_diag_pos,
+ "duplicate pool '" + pool->name() + "'", err);
+ }
return true;
}
-bool ManifestParser::ParseDefault(string* err) {
- EvalString eval;
- if (!lexer_.ReadPath(&eval, err))
- return false;
- if (eval.empty())
- return lexer_.Error("expected target name", err);
-
- do {
- string path = eval.Evaluate(env_);
- string path_err;
- uint64_t slash_bits; // Unused because this only does lookup.
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- if (!state_->AddDefault(path, &path_err))
- return lexer_.Error(path_err, err);
-
- eval.Clear();
- if (!lexer_.ReadPath(&eval, err))
- return false;
- } while (!eval.empty());
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
-
- return true;
-}
-
-bool ManifestParser::ParseEdge(string* err) {
- vector<EvalString> ins, outs;
-
+bool DfsParser::HandleClump(Clump* clump, const LoadedFile& file, Scope* scope,
+ std::string* err) {
+ METRIC_RECORD(".ninja load : scope setup");
+ // Allocate DFS and scope positions for the clump.
+ clump->pos_.scope = scope->AllocDecls(clump->decl_count());
+ clump->pos_.dfs_location = state_->AllocDfsLocation(clump->decl_count());
{
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
- return false;
- while (!out.empty()) {
- outs.push_back(out);
-
- out.Clear();
- if (!lexer_.ReadPath(&out, err))
- return false;
+ METRIC_RECORD(".ninja load : scope setup : bindings");
+ for (Binding* binding : clump->bindings_) {
+ scope->AddBinding(binding);
}
}
-
- // Add all implicit outs, counting how many as we go.
- int implicit_outs = 0;
- if (lexer_.PeekToken(Lexer::PIPE)) {
- for (;;) {
- EvalString out;
- if (!lexer_.ReadPath(&out, err))
- return err;
- if (out.empty())
- break;
- outs.push_back(out);
- ++implicit_outs;
- }
- }
-
- if (outs.empty())
- return lexer_.Error("expected path", err);
-
- if (!ExpectToken(Lexer::COLON, err))
- return false;
-
- string rule_name;
- if (!lexer_.ReadIdent(&rule_name))
- return lexer_.Error("expected build command name", err);
-
- const Rule* rule = env_->LookupRule(rule_name);
- if (!rule)
- return lexer_.Error("unknown build rule '" + rule_name + "'", err);
-
- for (;;) {
- // XXX should we require one path here?
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return false;
- if (in.empty())
- break;
- ins.push_back(in);
- }
-
- // Add all implicit deps, counting how many as we go.
- int implicit = 0;
- if (lexer_.PeekToken(Lexer::PIPE)) {
- for (;;) {
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return err;
- if (in.empty())
- break;
- ins.push_back(in);
- ++implicit;
- }
- }
-
- // Add all order-only deps, counting how many as we go.
- int order_only = 0;
- if (lexer_.PeekToken(Lexer::PIPE2)) {
- for (;;) {
- EvalString in;
- if (!lexer_.ReadPath(&in, err))
- return false;
- if (in.empty())
- break;
- ins.push_back(in);
- ++order_only;
- }
- }
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
-
- // Bindings on edges are rare, so allocate per-edge envs only when needed.
- bool has_indent_token = lexer_.PeekToken(Lexer::INDENT);
- BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_;
- while (has_indent_token) {
- string key;
- EvalString val;
- if (!ParseLet(&key, &val, err))
- return false;
-
- env->AddBinding(key, val.Evaluate(env_));
- has_indent_token = lexer_.PeekToken(Lexer::INDENT);
- }
-
- Edge* edge = state_->AddEdge(rule);
- edge->env_ = env;
-
- string pool_name = edge->GetBinding("pool");
- if (!pool_name.empty()) {
- Pool* pool = state_->LookupPool(pool_name);
- if (pool == NULL)
- return lexer_.Error("unknown pool name '" + pool_name + "'", err);
- edge->pool_ = pool;
- }
-
- edge->outputs_.reserve(outs.size());
- for (size_t i = 0, e = outs.size(); i != e; ++i) {
- string path = outs[i].Evaluate(env);
- string path_err;
- uint64_t slash_bits;
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- if (!state_->AddOut(edge, path, slash_bits)) {
- if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
- lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]",
- err);
- return false;
- } else {
- if (!quiet_) {
- Warning("multiple rules generate %s. "
- "builds involving this target will not be correct; "
- "continuing anyway [-w dupbuild=warn]",
- path.c_str());
- }
- if (e - i <= static_cast<size_t>(implicit_outs))
- --implicit_outs;
+ {
+ METRIC_RECORD(".ninja load : scope setup : rules");
+ for (Rule* rule : clump->rules_) {
+ if (!scope->AddRule(rule)) {
+ return DecorateError(file, rule->parse_state_.rule_name_diag_pos,
+ "duplicate rule '" + rule->name() + "'", err);
}
}
}
- if (edge->outputs_.empty()) {
- // All outputs of the edge are already created by other edges. Don't add
- // this edge. Do this check before input nodes are connected to the edge.
- state_->edges_.pop_back();
- delete edge;
- return true;
+ for (Pool* pool : clump->pools_) {
+ if (!HandlePool(pool, file, err)) {
+ return false;
+ }
}
- edge->implicit_outs_ = implicit_outs;
+ return true;
+}
- edge->inputs_.reserve(ins.size());
- for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
- string path = i->Evaluate(env);
- string path_err;
- uint64_t slash_bits;
- if (!CanonicalizePath(&path, &slash_bits, &path_err))
- return lexer_.Error(path_err, err);
- state_->AddIn(edge, path, slash_bits);
+bool DfsParser::LoadManifestTree(const LoadedFile& file, Scope* scope,
+ std::vector<Clump*>* out_clumps,
+ std::string* err) {
+ std::vector<ParserItem> items = ParseManifestChunks(file, thread_pool_);
+ ReserveSpaceInScopeTables(scope, items);
+
+ // With the chunks parsed, do a depth-first parse of the ninja manifest using
+ // the results of the parallel parse.
+ for (const auto& item : items) {
+ switch (item.kind) {
+ case ParserItem::kError:
+ *err = item.u.error->msg_;
+ return false;
+
+ case ParserItem::kRequiredVersion:
+ HandleRequiredVersion(*item.u.required_version, scope);
+ break;
+
+ case ParserItem::kInclude: {
+ const Include& include = *item.u.include;
+ const LoadedFile* child_file = nullptr;
+ Scope* child_scope = nullptr;
+ if (!HandleInclude(include, file, scope, &child_file, &child_scope, err))
+ return false;
+ if (!LoadManifestTree(*child_file, child_scope, out_clumps, err))
+ return false;
+ break;
+ }
+
+ case ParserItem::kClump:
+ if (!HandleClump(item.u.clump, file, scope, err))
+ return false;
+ out_clumps->push_back(item.u.clump);
+ break;
+
+ default:
+ assert(false && "unrecognized kind of ParserItem");
+ abort();
+ }
}
- edge->implicit_deps_ = implicit;
- edge->order_only_deps_ = order_only;
+ return true;
+}
+
+/// Parse an edge's path and add it to a vector on the Edge object.
+static inline bool AddPathToEdge(State* state, const Edge& edge,
+ std::vector<Node*>* out_vec,
+ const LoadedFile& file, Lexer& lexer,
+ std::string* err) {
+ HashedStrView key;
+ uint64_t slash_bits = 0;
+
+ StringPiece canon_path = lexer.PeekCanonicalPath();
+ if (!canon_path.empty()) {
+ key = canon_path;
+ } else {
+ LexedPath path;
+ if (!lexer.ReadPath(&path, err) || path.str_.empty()) {
+ assert(false && "manifest file apparently changed during parsing");
+ abort();
+ }
+
+ thread_local std::string tls_work_buf;
+ std::string& work_buf = tls_work_buf;
+ work_buf.clear();
+ EvaluatePathOnEdge(&work_buf, path, edge);
+
+ std::string path_err;
+ if (!CanonicalizePath(&work_buf, &slash_bits, &path_err)) {
+ return DecorateError(file, edge.parse_state_.final_diag_pos, path_err,
+ err);
+ }
+ key = work_buf;
+ }
+
+ Node* node = state->GetNode(key, 0);
+ node->UpdateFirstReference(edge.dfs_location(), slash_bits);
+ out_vec->push_back(node);
+ return true;
+}
+
+static const HashedStrView kPool { "pool" };
+static const HashedStrView kDeps { "deps" };
+
+struct ManifestLoader {
+private:
+ State* const state_ = nullptr;
+ ThreadPool* const thread_pool_ = nullptr;
+ const ManifestParserOptions options_;
+ const bool quiet_ = false;
+
+ bool AddEdgeToGraph(Edge* edge, const LoadedFile& file, std::string* err);
+
+ /// This function runs on a worker thread and adds a clump's declarations to
+ /// the build graph.
+ bool FinishAddingClumpToGraph(Clump* clump, std::string* err);
+
+ bool FinishLoading(const std::vector<Clump*>& clumps, std::string* err);
+
+public:
+ ManifestLoader(State* state, ThreadPool* thread_pool,
+ ManifestParserOptions options, bool quiet)
+ : state_(state), thread_pool_(thread_pool), options_(options),
+ quiet_(quiet) {}
+
+ bool Load(ManifestFileSet* file_set, const LoadedFile& root_manifest,
+ std::string* err);
+};
+
+bool ManifestLoader::AddEdgeToGraph(Edge* edge, const LoadedFile& file,
+ std::string* err) {
+
+ const ScopePosition edge_pos = edge->pos_.scope_pos();
+
+ // Look up the edge's rule.
+ edge->rule_ = Scope::LookupRuleAtPos(edge->parse_state_.rule_name, edge_pos);
+ if (edge->rule_ == nullptr) {
+ std::string msg = "unknown build rule '" +
+ edge->parse_state_.rule_name.AsString() + "'";
+ return DecorateError(file, edge->parse_state_.rule_name_diag_pos, msg, err);
+ }
+
+ // Now that the edge's bindings are available, check whether the edge has a
+ // pool. This check requires the full edge+rule evaluation system.
+ std::string pool_name;
+ if (!edge->EvaluateVariable(&pool_name, kPool, err, EdgeEval::kParseTime))
+ return false;
+ if (pool_name.empty()) {
+ edge->pool_ = &State::kDefaultPool;
+ } else {
+ edge->pool_ = state_->LookupPoolAtPos(pool_name, edge->pos_.dfs_location());
+ if (edge->pool_ == nullptr) {
+ return DecorateError(file, edge->parse_state_.final_diag_pos,
+ "unknown pool name '" + pool_name + "'", err);
+ }
+ }
+
+ edge->outputs_.reserve(edge->explicit_outs_ + edge->implicit_outs_);
+ edge->inputs_.reserve(edge->explicit_deps_ + edge->implicit_deps_ +
+ edge->order_only_deps_);
+ edge->validations_.reserve(edge->validation_deps_);
+
+ // Add the input and output nodes. We already lexed them in the first pass,
+ // but we couldn't add them because scope bindings weren't available. To save
+ // memory, the first pass only recorded the lexer position of each category
+ // of input/output nodes, rather than each path's location.
+ Lexer lexer(file.filename(), file.content(), file.content().data());
+ for (const Edge::DeferredPathList& path_list :
+ edge->parse_state_.deferred_path_lists) {
+ std::vector<Node*>* vec =
+ path_list.type == Edge::DeferredPathList::INPUT ? &edge->inputs_ :
+ path_list.type == Edge::DeferredPathList::OUTPUT ? &edge->outputs_ :
+ &edge->validations_;
+ lexer.ResetPos(path_list.lexer_pos);
+ for (int i = 0; i < path_list.count; ++i) {
+ if (!AddPathToEdge(state_, *edge, vec, file, lexer, err))
+ return false;
+ }
+ // Verify that there are no more paths to parse.
+ LexedPath path;
+ if (!lexer.ReadPath(&path, err) || !path.str_.empty()) {
+ assert(false && "manifest file apparently changed during parsing");
+ abort();
+ }
+ }
+
+ // This compatibility mode filters nodes from the edge->inputs_ list; do it
+ // before linking the edge inputs and nodes.
if (options_.phony_cycle_action_ == kPhonyCycleActionWarn &&
edge->maybe_phonycycle_diagnostic()) {
- // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements
- // that reference themselves. Ninja used to tolerate these in the
- // build graph but that has since been fixed. Filter them out to
- // support users of those old CMake versions.
+ // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements that
+ // reference themselves. Ninja used to tolerate these in the build graph
+ // but that has since been fixed. Filter them out to support users of those
+ // old CMake versions.
Node* out = edge->outputs_[0];
- vector<Node*>::iterator new_end =
- remove(edge->inputs_.begin(), edge->inputs_.end(), out);
+ std::vector<Node*>::iterator new_end =
+ std::remove(edge->inputs_.begin(), edge->inputs_.end(), out);
if (new_end != edge->inputs_.end()) {
edge->inputs_.erase(new_end, edge->inputs_.end());
+ --edge->explicit_deps_;
if (!quiet_) {
- Warning("phony target '%s' names itself as an input; "
- "ignoring [-w phonycycle=warn]",
- out->path().c_str());
+ Warning("phony target '%s' names itself as an input; ignoring "
+ "[-w phonycycle=warn]", out->path().c_str());
}
}
}
// Multiple outputs aren't (yet?) supported with depslog.
- string deps_type = edge->GetBinding("deps");
- if (!deps_type.empty() && edge->outputs_.size() > 1) {
- return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
- "bring this up on the mailing list if it affects you",
- err);
+ std::string deps_type;
+ if (!edge->EvaluateVariable(&deps_type, kDeps, err, EdgeEval::kParseTime))
+ return false;
+ if (!deps_type.empty() && edge->outputs_.size() - edge->implicit_outs_ > 1) {
+ return DecorateError(file, edge->parse_state_.final_diag_pos,
+ "multiple outputs aren't (yet?) supported by depslog; "
+ "bring this up on the mailing list if it affects you",
+ err);
}
return true;
}
-bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
- EvalString eval;
- if (!lexer_.ReadPath(&eval, err))
- return false;
- string path = eval.Evaluate(env_);
+bool ManifestLoader::FinishAddingClumpToGraph(Clump* clump, std::string* err) {
+ std::string work_buf;
- ManifestParser subparser(state_, file_reader_, options_);
- if (new_scope) {
- subparser.env_ = new BindingEnv(env_);
- } else {
- subparser.env_ = env_;
+ // Precompute all binding values. Discard each evaluated string -- we just
+ // need to make sure each binding's value isn't coming from the mmap'ed
+ // manifest anymore.
+ for (Binding* binding : clump->bindings_) {
+ work_buf.clear();
+ binding->Evaluate(&work_buf);
}
- if (!subparser.Load(path, err, &lexer_))
- return false;
-
- if (!ExpectToken(Lexer::NEWLINE, err))
- return false;
+ for (Edge* edge : clump->edges_) {
+ if (!AddEdgeToGraph(edge, clump->file_, err))
+ return false;
+ }
return true;
}
-bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) {
- Lexer::Token token = lexer_.ReadToken();
- if (token != expected) {
- string message = string("expected ") + Lexer::TokenName(expected);
- message += string(", got ") + Lexer::TokenName(token);
- message += Lexer::TokenErrorHint(expected);
- return lexer_.Error(message, err);
+bool ManifestLoader::FinishLoading(const std::vector<Clump*>& clumps,
+ std::string* err) {
+ {
+ // Most of this pass's time is spent adding the edges. (i.e. The time spent
+ // evaluating the bindings is negligible.)
+ METRIC_RECORD(".ninja load : edge setup");
+
+ size_t output_count = 0;
+ for (Clump* clump : clumps)
+ output_count += clump->edge_output_count_;
+
+ // Construct the initial graph of input/output nodes. Select an initial size
+ // that's likely to keep the number of collisions low. The number of edges'
+ // non-implicit outputs is a decent enough proxy for the final number of
+ // nodes. (I see acceptable performance even with a much lower number of
+ // buckets, e.g. 100 times fewer.)
+ state_->paths_.reserve(state_->paths_.size() + output_count * 3);
+
+ if (!PropagateError(err, ParallelMap(thread_pool_, clumps,
+ [this](Clump* clump) {
+ std::string err;
+ FinishAddingClumpToGraph(clump, &err);
+ return err;
+ }))) {
+ return false;
+ }
}
+ {
+ // Record the in-edge for each node that's built by an edge. Detect
+ // duplicate edges.
+ //
+ // With dupbuild=warn (the default until 1.9.0), when two edges generate the
+ // same node, remove the duplicate node from the output list of the later
+ // edge. If all of an edge's outputs are removed, remove the edge from the
+ // graph.
+ METRIC_RECORD(".ninja load : link edge outputs");
+ for (Clump* clump : clumps) {
+ for (size_t edge_idx = 0; edge_idx < clump->edges_.size(); ) {
+ // Scan all Edge outputs and link them to the Node objects.
+ Edge* edge = clump->edges_[edge_idx];
+ for (size_t i = 0; i < edge->outputs_.size(); ) {
+ Node* output = edge->outputs_[i];
+ if (output->in_edge() == nullptr) {
+ output->set_in_edge(edge);
+ ++i;
+ continue;
+ }
+ // Two edges produce the same output node.
+ if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
+ return DecorateError(clump->file_,
+ edge->parse_state_.final_diag_pos,
+ "multiple rules generate " + output->path() +
+ " [-w dupbuild=err]", err);
+ } else {
+ if (!quiet_) {
+ Warning("multiple rules generate %s. "
+ "builds involving this target will not be correct; "
+ "continuing anyway [-w dupbuild=warn]",
+ output->path().c_str());
+ }
+ if (edge->is_implicit_out(i))
+ --edge->implicit_outs_;
+ else
+ --edge->explicit_outs_;
+ edge->outputs_.erase(edge->outputs_.begin() + i);
+ }
+ }
+ if (edge->outputs_.empty()) {
+ // All outputs of the edge are already created by other edges. Remove
+ // this edge from the graph. This removal happens before the edge's
+ // inputs are linked to nodes.
+ clump->edges_.erase(clump->edges_.begin() + edge_idx);
+ continue;
+ }
+ ++edge_idx;
+ }
+ }
+ }
+ {
+ // Now that all invalid edges are removed from the graph, record an out-edge
+ // on each node that's needed by an edge.
+ METRIC_RECORD(".ninja load : link edge inputs");
+ ParallelMap(thread_pool_, clumps, [](Clump* clump) {
+ for (Edge* edge : clump->edges_) {
+ for (Node* input : edge->inputs_) {
+ input->AddOutEdge(edge);
+ }
+ for (Node* validation : edge->validations_) {
+ validation->AddValidationOutEdge(edge);
+ }
+ }
+ });
+ }
+ {
+ METRIC_RECORD(".ninja load : default targets");
+
+ for (Clump* clump : clumps) {
+ for (DefaultTarget* target : clump->default_targets_) {
+ std::string path;
+ EvaluatePathInScope(&path, target->parsed_path_,
+ target->pos_.scope_pos());
+ uint64_t slash_bits; // Unused because this only does lookup.
+ std::string path_err;
+ if (!CanonicalizePath(&path, &slash_bits, &path_err))
+ return DecorateError(clump->file_, target->diag_pos_, path_err, err);
+
+ Node* node = state_->LookupNodeAtPos(path, target->pos_.dfs_location());
+ if (node == nullptr) {
+ return DecorateError(clump->file_, target->diag_pos_,
+ "unknown target '" + path + "'", err);
+ }
+ state_->AddDefault(node);
+ }
+ }
+ }
+ {
+ // Add the clump edges into the global edge vector, and assign edge IDs.
+ // Edge IDs are used for custom protobuf-based Ninja frontends. An edge's ID
+ // is equal to its index in the global edge vector, so delay the assignment
+ // of edge IDs until we've removed duplicate edges above (dupbuild=warn).
+
+ METRIC_RECORD(".ninja load : build edge table");
+
+ // Copy edges to the global edge table.
+ size_t old_size = state_->edges_.size();
+ size_t new_size = old_size;
+ for (Clump* clump : clumps) {
+ new_size += clump->edges_.size();
+ }
+ state_->edges_.reserve(new_size);
+ for (Clump* clump : clumps) {
+ std::copy(clump->edges_.begin(), clump->edges_.end(),
+ std::back_inserter(state_->edges_));
+ }
+ // Assign edge IDs.
+ ParallelMap(thread_pool_, IntegralRange<size_t>(old_size, new_size),
+ [this](size_t idx) {
+ state_->edges_[idx]->id_ = idx;
+ });
+ }
+
return true;
}
+
+bool ManifestLoader::Load(ManifestFileSet* file_set,
+ const LoadedFile& root_manifest, std::string* err) {
+ DfsParser dfs_parser(file_set, state_, thread_pool_);
+ std::vector<Clump*> clumps;
+ if (!dfs_parser.LoadManifestTree(root_manifest, &state_->root_scope_, &clumps,
+ err)) {
+ return false;
+ }
+ return FinishLoading(clumps, err);
+}
+
+} // anonymous namespace
+
+bool ManifestParser::Load(const string& filename, string* err) {
+ METRIC_RECORD(".ninja load");
+
+ ManifestFileSet file_set(file_reader_);
+ const LoadedFile* file = nullptr;
+ if (!file_set.LoadFile(filename, &file, err))
+ return false;
+
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+ ManifestLoader loader(state_, thread_pool.get(), options_, false);
+ return loader.Load(&file_set, *file, err);
+}
+
+bool ManifestParser::ParseTest(const string& input, string* err) {
+ ManifestFileSet file_set(file_reader_);
+ std::unique_ptr<ThreadPool> thread_pool = CreateThreadPool();
+ ManifestLoader loader(state_, thread_pool.get(), options_, true);
+ return loader.Load(&file_set, HeapLoadedFile("input", input), err);
+}
diff --git a/src/manifest_parser.h b/src/manifest_parser.h
index 2136018..3f1895d 100644
--- a/src/manifest_parser.h
+++ b/src/manifest_parser.h
@@ -1,4 +1,4 @@
-// Copyright 2011 Google Inc. All Rights Reserved.
+// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -17,12 +17,6 @@
#include <string>
-using namespace std;
-
-#include "lexer.h"
-
-struct BindingEnv;
-struct EvalString;
struct FileReader;
struct State;
@@ -37,51 +31,30 @@
};
struct ManifestParserOptions {
- ManifestParserOptions()
- : dupe_edge_action_(kDupeEdgeActionWarn),
- phony_cycle_action_(kPhonyCycleActionWarn) {}
- DupeEdgeAction dupe_edge_action_;
- PhonyCycleAction phony_cycle_action_;
+ DupeEdgeAction dupe_edge_action_ = kDupeEdgeActionWarn;
+ PhonyCycleAction phony_cycle_action_ = kPhonyCycleActionWarn;
};
-/// Parses .ninja files.
struct ManifestParser {
ManifestParser(State* state, FileReader* file_reader,
- ManifestParserOptions options = ManifestParserOptions());
+ ManifestParserOptions options = ManifestParserOptions())
+ : state_(state),
+ file_reader_(file_reader),
+ options_(options) {}
/// Load and parse a file.
- bool Load(const string& filename, string* err, Lexer* parent = NULL);
+ bool Load(const std::string& filename, std::string* err);
- /// Parse a text string of input. Used by tests.
- bool ParseTest(const string& input, string* err) {
- quiet_ = true;
- return Parse("input", input, err);
- }
+ /// Parse a text string of input. Used by tests.
+ ///
+ /// Some tests may call ParseTest multiple times with the same State object.
+ /// Each call adds to the previous state; it doesn't replace it.
+ bool ParseTest(const std::string& input, std::string* err);
private:
- /// Parse a file, given its contents as a string.
- bool Parse(const string& filename, const string& input, string* err);
-
- /// Parse various statement types.
- bool ParsePool(string* err);
- bool ParseRule(string* err);
- bool ParseLet(string* key, EvalString* val, string* err);
- bool ParseEdge(string* err);
- bool ParseDefault(string* err);
-
- /// Parse either a 'subninja' or 'include' line.
- bool ParseFileInclude(bool new_scope, string* err);
-
- /// If the next token is not \a expected, produce an error string
- /// saying "expectd foo, got bar".
- bool ExpectToken(Lexer::Token expected, string* err);
-
- State* state_;
- BindingEnv* env_;
- FileReader* file_reader_;
- Lexer lexer_;
+ State* state_ = nullptr;
+ FileReader* file_reader_ = nullptr;
ManifestParserOptions options_;
- bool quiet_;
};
#endif // NINJA_MANIFEST_PARSER_H_
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index c91d8d1..0732129 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -48,11 +48,10 @@
"\n"
"build result: cat in_1.cc in-2.O\n"));
- ASSERT_EQ(3u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(3u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat", rule->name());
- EXPECT_EQ("[cat ][$in][ > ][$out]",
- rule->GetBinding("command")->Serialize());
+ EXPECT_EQ("cat $in > $out\n", *rule->GetBinding("command"));
}
TEST_F(ParserTest, RuleAttributes) {
@@ -63,6 +62,7 @@
" depfile = a\n"
" deps = a\n"
" description = a\n"
+" phony_output = a\n"
" generator = a\n"
" restat = a\n"
" rspfile = a\n"
@@ -81,12 +81,12 @@
"build result: cat in_1.cc in-2.O\n"
" #comment\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat", rule->name());
Edge* edge = state.GetNode("result", 0)->in_edge();
- EXPECT_TRUE(edge->GetBindingBool("restat"));
- EXPECT_FALSE(edge->GetBindingBool("generator"));
+ EXPECT_TRUE(edge->IsRestat());
+ EXPECT_FALSE(edge->IsGenerator());
}
TEST_F(ParserTest, IgnoreIndentedBlankLines) {
@@ -101,7 +101,7 @@
"variable=1\n"));
// the variable must be in the top level environment
- EXPECT_EQ("1", state.bindings_.LookupVariable("variable"));
+ EXPECT_EQ("1", state.root_scope_.LookupVariable("variable"));
}
TEST_F(ParserTest, ResponseFiles) {
@@ -114,13 +114,12 @@
"build out: cat_rsp in\n"
" rspfile=out.rsp\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat_rsp", rule->name());
- EXPECT_EQ("[cat ][$rspfile][ > ][$out]",
- rule->GetBinding("command")->Serialize());
- EXPECT_EQ("[$rspfile]", rule->GetBinding("rspfile")->Serialize());
- EXPECT_EQ("[$in]", rule->GetBinding("rspfile_content")->Serialize());
+ EXPECT_EQ("cat $rspfile > $out\n", *rule->GetBinding("command"));
+ EXPECT_EQ("$rspfile\n", *rule->GetBinding("rspfile"));
+ EXPECT_EQ("$in\n", *rule->GetBinding("rspfile_content"));
}
TEST_F(ParserTest, InNewline) {
@@ -131,11 +130,10 @@
"build out: cat_rsp in in2\n"
" rspfile=out.rsp\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("cat_rsp", rule->name());
- EXPECT_EQ("[cat ][$in_newline][ > ][$out]",
- rule->GetBinding("command")->Serialize());
+ EXPECT_EQ("cat $in_newline > $out\n", *rule->GetBinding("command"));
Edge* edge = state.edges_[0];
EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand());
@@ -159,7 +157,7 @@
Edge* edge = state.edges_[0];
EXPECT_EQ("ld one-letter-test -pthread -under -o a b c",
edge->EvaluateCommand());
- EXPECT_EQ("1/2", state.bindings_.LookupVariable("nested2"));
+ EXPECT_EQ("1/2", state.root_scope_.LookupVariable("nested2"));
edge = state.edges_[1];
EXPECT_EQ("ld one-letter-test 1/2/3 -under -o supernested x",
@@ -192,10 +190,12 @@
"build a: link c $\n"
" d e f\n"));
- ASSERT_EQ(2u, state.bindings_.GetRules().size());
- const Rule* rule = state.bindings_.GetRules().begin()->second;
+ ASSERT_EQ(2u, state.root_scope_.GetRules().size());
+ const Rule* rule = state.root_scope_.GetRules().begin()->second;
EXPECT_EQ("link", rule->name());
- EXPECT_EQ("[foo bar baz]", rule->GetBinding("command")->Serialize());
+ std::string command = *rule->GetBinding("command");
+ EXPECT_EQ("foo bar $\n baz\n", command);
+ EXPECT_EQ("foo bar baz", EvaluateBindingForTesting(command));
}
TEST_F(ParserTest, Backslash) {
@@ -203,15 +203,15 @@
"foo = bar\\baz\n"
"foo2 = bar\\ baz\n"
));
- EXPECT_EQ("bar\\baz", state.bindings_.LookupVariable("foo"));
- EXPECT_EQ("bar\\ baz", state.bindings_.LookupVariable("foo2"));
+ EXPECT_EQ("bar\\baz", state.root_scope_.LookupVariable("foo"));
+ EXPECT_EQ("bar\\ baz", state.root_scope_.LookupVariable("foo2"));
}
TEST_F(ParserTest, Comment) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
"# this is a comment\n"
"foo = not # a comment\n"));
- EXPECT_EQ("not # a comment", state.bindings_.LookupVariable("foo"));
+ EXPECT_EQ("not # a comment", state.root_scope_.LookupVariable("foo"));
}
TEST_F(ParserTest, Dollars) {
@@ -222,7 +222,7 @@
"x = $$dollar\n"
"build $x: foo y\n"
));
- EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x"));
+ EXPECT_EQ("$dollar", state.root_scope_.LookupVariable("x"));
#ifdef _WIN32
EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand());
#else
@@ -418,6 +418,38 @@
"default subninja\n"));
}
+TEST_F(ParserTest, NulCharErrors) {
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("\0"_s, &err));
+ EXPECT_EQ("input:1: unexpected NUL byte\n", err);
+ }
+
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("build foo\0"_s, &err));
+ EXPECT_EQ("input:1: unexpected NUL byte\n"
+ "build foo\n"
+ " ^ near here"
+ , err);
+ }
+
+ {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("foo\0"_s, &err));
+ EXPECT_EQ("input:1: expected '=', got nul byte\n"
+ "foo\n"
+ " ^ near here"
+ , err);
+ }
+}
+
TEST_F(ParserTest, Errors) {
{
State local_state;
@@ -707,10 +739,15 @@
}
{
+ // The default statement's target must be listed earlier in a build
+ // statement.
State local_state;
ManifestParser parser(&local_state, NULL);
string err;
- EXPECT_FALSE(parser.ParseTest("default nonexistent\n",
+ EXPECT_FALSE(parser.ParseTest("default nonexistent\n"
+ "rule cat\n"
+ " command = cat $in > $out\n"
+ "build nonexistent: cat existent\n",
&err));
EXPECT_EQ("input:1: unknown target 'nonexistent'\n"
"default nonexistent\n"
@@ -792,7 +829,8 @@
string err;
EXPECT_FALSE(parser.ParseTest("pool foo\n"
" depth = 4\n"
- "pool foo\n", &err));
+ "pool foo\n"
+ " depth = 2\n", &err));
EXPECT_EQ("input:3: duplicate pool 'foo'\n"
"pool foo\n"
" ^ near here"
@@ -854,6 +892,16 @@
EXPECT_EQ("", err);
}
+TEST_F(ParserTest, MultipleImplicitOutputsWithDeps) {
+ State local_state;
+ ManifestParser parser(&local_state, NULL);
+ string err;
+ EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n"
+ "build a.o | a.gcno: cc c.cc\n",
+ &err));
+ EXPECT_EQ("", err);
+}
+
TEST_F(ParserTest, MultipleOutputsWithDeps) {
State local_state;
ManifestParser parser(&local_state, NULL);
@@ -932,7 +980,7 @@
ASSERT_EQ(1u, fs_.files_read_.size());
EXPECT_EQ("include.ninja", fs_.files_read_[0]);
- EXPECT_EQ("inner", state.bindings_.LookupVariable("var"));
+ EXPECT_EQ("inner", state.root_scope_.LookupVariable("var"));
}
TEST_F(ParserTest, BrokenInclude) {
@@ -965,6 +1013,16 @@
ASSERT_TRUE(edge->is_order_only(1));
}
+TEST_F(ParserTest, Validations) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n command = cat $in > $out\n"
+"build foo: cat bar |@ baz\n"));
+
+ Edge* edge = state.LookupNode("foo")->in_edge();
+ ASSERT_EQ(edge->validations_.size(), 1);
+ EXPECT_EQ(edge->validations_[0]->path(), "baz");
+}
+
TEST_F(ParserTest, ImplicitOutput) {
ASSERT_NO_FATAL_FAILURE(AssertParse(
"rule cat\n"
@@ -1085,3 +1143,67 @@
" description = YAY!\r\n",
&err));
}
+
+TEST_F(ParserTest, IncludeUsingAVariable) {
+ // Each include statement should use the $path binding from the previous
+ // manifest declarations.
+ fs_.Create("foo.ninja", "path = bar.ninja\n"
+ "include $path\n");
+ fs_.Create("bar.ninja", "");
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"path = foo.ninja\n"
+"include $path\n"
+"include $path\n"
+"path = nonexistent.ninja\n"));
+}
+
+TEST_F(ParserTest, UnscopedPool) {
+ // Pools aren't scoped.
+ fs_.Create("foo.ninja", "pool link\n"
+ " depth = 3\n");
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n"
+" command = cat $in > $out\n"
+" pool = link\n"
+"subninja foo.ninja\n"
+"build a: cat b\n"));
+}
+
+TEST_F(ParserTest, PoolDeclaredAfterUse) {
+ // A pool must be declared before an edge that uses it.
+ ManifestParser parser(&state, nullptr);
+ std::string err;
+ EXPECT_FALSE(parser.ParseTest("rule cat\n"
+ " command = cat $in > $out\n"
+ " pool = link\n"
+ "build a: cat b\n"
+ "pool link\n"
+ " depth = 3\n", &err));
+ EXPECT_EQ("input:5: unknown pool name 'link'\n", err);
+}
+
+TEST_F(ParserTest, DefaultReferencesEdgeInput) {
+ // The default statement's target must be listed before the default statement.
+ // It's OK if the target's first reference is an input, though.
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"rule cat\n command = cat $in > $out\n"
+"build a1: cat b1\n"
+"build a2: cat b2\n"
+"default b1 b2\n"
+"build b1: cat c1\n"));
+}
+
+TEST_F(ParserTest, SelfVarExpansion) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(
+"var = xxx\n"
+"var1 = ${var}\n"
+"var = a${var}a\n"
+"var2 = ${var}\n"
+"var = b${var}b\n"
+"var3 = ${var}\n"
+"var = c${var}c\n"));
+ EXPECT_EQ("xxx", state.root_scope_.LookupVariable("var1"));
+ EXPECT_EQ("axxxa", state.root_scope_.LookupVariable("var2"));
+ EXPECT_EQ("baxxxab", state.root_scope_.LookupVariable("var3"));
+ EXPECT_EQ("cbaxxxabc", state.root_scope_.LookupVariable("var"));
+}
diff --git a/src/metrics.cc b/src/metrics.cc
index a7d3c7a..5e4cef9 100644
--- a/src/metrics.cc
+++ b/src/metrics.cc
@@ -19,13 +19,17 @@
#include <string.h>
#ifndef _WIN32
+#include <sys/resource.h>
#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
#else
#include <windows.h>
#endif
#include <algorithm>
+#include "status.h"
#include "util.h"
Metrics* g_metrics = NULL;
@@ -75,45 +79,40 @@
} // anonymous namespace
-ScopedMetric::ScopedMetric(Metric* metric) {
- metric_ = metric;
- if (!metric_)
- return;
+void ScopedMetric::RecordStart() {
start_ = HighResTimer();
}
-ScopedMetric::~ScopedMetric() {
- if (!metric_)
- return;
- metric_->count++;
- int64_t dt = TimerToMicros(HighResTimer() - start_);
- metric_->sum += dt;
+
+void ScopedMetric::RecordResult() {
+ int64_t duration = TimerToMicros(HighResTimer() - start_);
+ metric_->AddResult(1, duration);
}
Metric* Metrics::NewMetric(const string& name) {
- Metric* metric = new Metric;
- metric->name = name;
- metric->count = 0;
- metric->sum = 0;
- metrics_.push_back(metric);
- return metric;
+ std::lock_guard<std::mutex> lock(mutex_);
+ Metric* result = new Metric(name);
+ metrics_.push_back(result);
+ return result;
}
-void Metrics::Report() {
+void Metrics::Report(Status *status) {
+ std::lock_guard<std::mutex> lock(mutex_);
+
int width = 0;
for (vector<Metric*>::iterator i = metrics_.begin();
i != metrics_.end(); ++i) {
- width = max((int)(*i)->name.size(), width);
+ width = max((int)(*i)->name().size(), width);
}
- printf("%-*s\t%-6s\t%-9s\t%s\n", width,
+ status->Debug("%-*s\t%-6s\t%-9s\t%s", width,
"metric", "count", "avg (us)", "total (ms)");
for (vector<Metric*>::iterator i = metrics_.begin();
i != metrics_.end(); ++i) {
Metric* metric = *i;
- double total = metric->sum / (double)1000;
- double avg = metric->sum / (double)metric->count;
- printf("%-*s\t%-6d\t%-8.1f\t%.1f\n", width, metric->name.c_str(),
- metric->count, avg, total);
+ double total = metric->time() / (double)1000;
+ double avg = metric->time() / (double)metric->count();
+ status->Debug("%-*s\t%-6d\t%-8.1f\t%.1f", width, metric->name().c_str(),
+ metric->count(), avg, total);
}
}
@@ -125,3 +124,39 @@
return TimerToMicros(HighResTimer()) / 1000;
}
+void DumpMemoryUsage(Status *status) {
+#if defined(__linux__)
+ std::vector<std::string> words;
+ struct rusage usage {};
+ if (getrusage(RUSAGE_SELF, &usage) == 0) {
+ words.push_back(std::to_string(usage.ru_majflt) + " maj faults");
+ words.push_back(std::to_string(usage.ru_minflt) + " min faults");
+ words.push_back(std::to_string(usage.ru_maxrss / 1024) + " MiB maxrss");
+ }
+ char status_path[256];
+ snprintf(status_path, sizeof(status_path), "/proc/%d/status",
+ static_cast<int>(getpid()));
+ if (FILE* status_fp = fopen(status_path, "r")) {
+ char* line = nullptr;
+ size_t n = 0;
+ while (getline(&line, &n, status_fp) > 0) {
+ unsigned long rss = 0;
+ if (sscanf(line, "VmRSS:\t%lu kB\n", &rss) == 1) {
+ words.push_back(std::to_string(rss / 1024) + " MiB rss");
+ }
+ }
+ free(line);
+ fclose(status_fp);
+ }
+ if (!words.empty()) {
+ string final;
+ for (size_t i = 0; i < words.size(); ++i) {
+ if (i > 0) {
+ final += ", ";
+ }
+ final += words[i];
+ }
+ status->Debug("%s", final.c_str());
+ }
+#endif
+}
diff --git a/src/metrics.h b/src/metrics.h
index b6da859..48f26d1 100644
--- a/src/metrics.h
+++ b/src/metrics.h
@@ -15,32 +15,83 @@
#ifndef NINJA_METRICS_H_
#define NINJA_METRICS_H_
+#include <atomic>
+#include <mutex>
#include <string>
#include <vector>
using namespace std;
#include "util.h" // For int64_t.
+struct Status;
+
/// The Metrics module is used for the debug mode that dumps timing stats of
/// various actions. To use, see METRIC_RECORD below.
/// A single metrics we're tracking, like "depfile load time".
struct Metric {
- string name;
- /// Number of times we've hit the code path.
- int count;
- /// Total time (in micros) we've spent on the code path.
- int64_t sum;
-};
+ Metric(const std::string& name) : name_(name) {}
+ void AddResult(int count, int64_t time_us) {
+ thread_local size_t this_slot = GetThreadSlotIndex();
+ Slot& slot = slots_[this_slot];
+ slot.count += count;
+ slot.time += time_us;
+ }
-/// A scoped object for recording a metric across the body of a function.
-/// Used by the METRIC_RECORD macro.
-struct ScopedMetric {
- explicit ScopedMetric(Metric* metric);
- ~ScopedMetric();
+ const std::string& name() const { return name_; }
+
+ int count() const {
+ int result = 0;
+ for (const Slot& slot : slots_) {
+ result += slot.count;
+ }
+ return result;
+ }
+
+ int64_t time() const {
+ int64_t result = 0;
+ for (const Slot& slot : slots_) {
+ result += slot.time;
+ }
+ return result;
+ }
private:
+ /// Try to give each thread a different slot to reduce thread contention.
+ struct NINJA_ALIGNAS_CACHE_LINE Slot {
+ /// Number of times we've hit the code path.
+ std::atomic<int> count {};
+ /// Total time (in micros) we've spent on the code path.
+ std::atomic<int64_t> time {};
+ };
+
+ std::string name_;
+ std::vector<Slot> slots_ { GetThreadSlotCount() };
+};
+
+/// A scoped object for recording a metric across the body of a function.
+/// Used by the METRIC_RECORD macro. Inline the metric_ null check to minimize
+/// the effect on the typical case where metrics are disabled.
+struct ScopedMetric {
+ explicit ScopedMetric(Metric* metric) : metric_(metric) {
+ if (metric_ == nullptr) {
+ return;
+ }
+ RecordStart();
+ }
+
+ ~ScopedMetric() {
+ if (metric_ == nullptr) {
+ return;
+ }
+ RecordResult();
+ }
+
+private:
+ void RecordStart();
+ void RecordResult();
+
Metric* metric_;
/// Timestamp when the measurement started.
/// Value is platform-dependent.
@@ -52,9 +103,10 @@
Metric* NewMetric(const string& name);
/// Print a summary report to stdout.
- void Report();
+ void Report(Status *status);
private:
+ std::mutex mutex_;
vector<Metric*> metrics_;
};
@@ -89,4 +141,6 @@
extern Metrics* g_metrics;
+void DumpMemoryUsage(Status *status);
+
#endif // NINJA_METRICS_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index b608426..ac5cb7e 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -30,6 +30,9 @@
#include <unistd.h>
#endif
+#include <deque>
+#include <unordered_map>
+
#include "browse.h"
#include "build.h"
#include "build_log.h"
@@ -42,6 +45,8 @@
#include "manifest_parser.h"
#include "metrics.h"
#include "state.h"
+#include "status.h"
+#include "thread_pool.h"
#include "util.h"
#include "version.h"
@@ -53,6 +58,11 @@
void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep);
#endif
+// Ninja intentionally leaks memory. Turn off LeakSanitizer by default.
+extern "C" const char* __asan_default_options() {
+ return "detect_leaks=0";
+}
+
namespace {
struct Tool;
@@ -77,13 +87,17 @@
/// Whether a depfile with multiple targets on separate lines should
/// warn or print an error.
bool depfile_distinct_target_lines_should_err;
+
+ /// Whether to remain persistent.
+ bool persistent;
};
/// The Ninja main() loads up a series of data structures; various tools need
/// to poke into these, so store them as fields on an object.
struct NinjaMain : public BuildLogUser {
NinjaMain(const char* ninja_command, const BuildConfig& config) :
- ninja_command_(ninja_command), config_(config) {}
+ ninja_command_(ninja_command), config_(config),
+ start_time_millis_(GetTimeMillis()) {}
/// Command line used to run Ninja.
const char* ninja_command_;
@@ -116,6 +130,8 @@
// The various subcommands, run via "-t XXX".
int ToolGraph(const Options* options, int argc, char* argv[]);
+ int ToolPath(const Options* options, int argc, char* argv[]);
+ int ToolInputs(const Options* options, int argc, char* argv[]);
int ToolQuery(const Options* options, int argc, char* argv[]);
int ToolDeps(const Options* options, int argc, char* argv[]);
int ToolBrowse(const Options* options, int argc, char* argv[]);
@@ -142,14 +158,14 @@
/// Rebuild the manifest, if necessary.
/// Fills in \a err on error.
/// @return true if the manifest was rebuilt.
- bool RebuildManifest(const char* input_file, string* err);
+ bool RebuildManifest(const char* input_file, string* err, Status* status);
/// Build the targets listed on the command line.
/// @return an exit code.
- int RunBuild(int argc, char** argv);
+ int RunBuild(int argc, char** argv, Status* status);
/// Dump the output requested by '-d stats'.
- void DumpMetrics();
+ void DumpMetrics(Status *status);
virtual bool IsPathDead(StringPiece s) const {
Node* n = state_.LookupNode(s);
@@ -165,11 +181,13 @@
// Do keep entries around for files which still exist on disk, for
// generators that want to use this information.
string err;
- TimeStamp mtime = disk_interface_.Stat(s.AsString(), &err);
+ TimeStamp mtime = disk_interface_.LStat(s.AsString(), nullptr, &err);
if (mtime == -1)
Error("%s", err.c_str()); // Log and ignore Stat() errors.
return mtime == 0;
}
+
+ int64_t start_time_millis_;
};
/// Subtools, accessible via "-t foo".
@@ -219,8 +237,14 @@
" -d MODE enable debugging (use '-d list' to list modes)\n"
" -t TOOL run a subtool (use '-t list' to list subtools)\n"
" terminates toplevel options; further flags are passed to the tool\n"
-" -w FLAG adjust warnings (use '-w list' to list warnings)\n",
- kNinjaVersion, config.parallelism);
+" -o FLAG adjust options (use '-o list' to list options)\n"
+" -w FLAG adjust warnings (use '-w list' to list warnings)\n"
+#ifndef _WIN32
+"\n"
+" --frontend COMMAND execute COMMAND and pass serialized build output to it\n"
+" --frontend_file FILE write serialized build output to FILE\n"
+#endif
+ , kNinjaVersion, config.parallelism);
}
/// Choose a default value for the -j (parallelism) flag.
@@ -238,7 +262,8 @@
/// Rebuild the build manifest, if necessary.
/// Returns true if the manifest was rebuilt.
-bool NinjaMain::RebuildManifest(const char* input_file, string* err) {
+bool NinjaMain::RebuildManifest(const char* input_file, string* err,
+ Status* status) {
string path = input_file;
uint64_t slash_bits; // Unused because this path is only used for lookup.
if (!CanonicalizePath(&path, &slash_bits, err))
@@ -247,8 +272,9 @@
if (!node)
return false;
- Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
- if (!builder.AddTarget(node, err))
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_,
+ status, start_time_millis_);
+ if (!builder.AddTargets({ node }, err))
return false;
if (builder.AlreadyUpToDate())
@@ -285,11 +311,11 @@
Node* node = state_.LookupNode(path);
if (node) {
if (first_dependent) {
- if (node->out_edges().empty()) {
+ if (!node->has_out_edge()) {
*err = "'" + path + "' has no out edge";
return NULL;
}
- Edge* edge = node->out_edges()[0];
+ Edge* edge = node->GetOutEdges()[0];
if (edge->outputs_.empty()) {
edge->Dump();
Fatal("edge has no outputs");
@@ -347,6 +373,156 @@
return 0;
}
+int NinjaMain::ToolPath(const Options* options, int argc, char* argv[]) {
+ if (argc != 2) {
+ Error("expected two targets to find a dependency chain between");
+ return 1;
+ }
+ std::string err;
+ Node* out = CollectTarget(argv[0], &err);
+ if (!out) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+ Node* in = CollectTarget(argv[1], &err);
+ if (!in) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ // BFS from "in", looking for "out". The map record a node's input that leads
+ // back to "in".
+ std::unordered_map<Node*, Node*> node_next;
+ std::deque<Node*> queue;
+ queue.push_back(in);
+ while (!queue.empty()) {
+ Node* node = queue[0];
+ queue.pop_front();
+ if (node == out) {
+ for (; node != nullptr; node = node_next[node]) {
+ printf("%s\n", node->path().c_str());
+ }
+ return 0;
+ }
+ for (Edge* edge : node->GetOutEdges()) {
+ for (Node* output : edge->outputs_) {
+ if (node_next.count(output) == 0) {
+ node_next[output] = node;
+ queue.push_back(output);
+ }
+ }
+ }
+ for (Edge* edge : node->GetValidationOutEdges()) {
+ for (Node* output : edge->outputs_) {
+ if (node_next.count(output) == 0) {
+ node_next[output] = node;
+ queue.push_back(output);
+ }
+ }
+ }
+ }
+ Error("%s does not depend on %s", out->path().c_str(), in->path().c_str());
+ return 1;
+}
+
+void ToolInputsProcessNodeDeps(Node* node, DepsLog* deps_log, bool leaf_only,
+ bool include_deps);
+
+void ToolInputsProcessNode(Node* input, DepsLog* deps_log, bool leaf_only,
+ bool include_deps) {
+ if (input->InputsChecked()) return;
+ input->MarkInputsChecked();
+
+ // Recursively process input edges, possibly printing this node here.
+ Edge* input_edge = input->in_edge();
+ if (input_edge == nullptr || !leaf_only) {
+ printf("%s\n", input->path().c_str());
+ }
+ if (input_edge) {
+ for (Node* input : input_edge->inputs_) {
+ ToolInputsProcessNode(input, deps_log, leaf_only, include_deps);
+ }
+ if (include_deps && input_edge->outputs_.size() > 0) {
+ // Check deps on the input edge's first output because deps log entries
+ // are only stored on the first output of an edge.
+ ToolInputsProcessNodeDeps(input_edge->outputs_[0], deps_log, leaf_only,
+ include_deps);
+ }
+ }
+}
+
+void ToolInputsProcessNodeDeps(Node* node, DepsLog* deps_log, bool leaf_only,
+ bool include_deps) {
+ // Print all of this node's deps from the deps log. This often includes files
+ // that are not known by the node's input edge.
+ if (deps_log->IsDepsEntryLiveFor(node)) {
+ if (DepsLog::Deps* deps = deps_log->GetDeps(node); deps != nullptr) {
+ for (int i = 0; i < deps->node_count; ++i) {
+ if (Node* dep = deps->nodes[i]; dep != nullptr) {
+ ToolInputsProcessNode(dep, deps_log, leaf_only, include_deps);
+ }
+ }
+ }
+ }
+}
+
+int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
+ // The inputs tool uses getopt, and expects argv[0] to contain the name of
+ // the tool, i.e. "inputs".
+ ++argc;
+ --argv;
+
+ bool leaf_only = true;
+ bool include_deps = false;
+
+ optind = 1;
+ int opt;
+ while ((opt = getopt(argc, argv, "idh")) != -1) {
+ switch (opt) {
+ case 'i':
+ leaf_only = false;
+ break;
+ case 'd':
+ include_deps = true;
+ break;
+ case 'h':
+ default:
+ printf(
+ "usage: ninja -t inputs [options] target [target...]\n\n"
+ "options:\n"
+ " -i Include intermediate inputs.\n"
+ " -d Include deps from the deps log file (.ninja_deps).\n");
+ return 1;
+ }
+ }
+ argv += optind;
+ argc -= optind;
+
+ vector<Node*> nodes;
+ string err;
+ if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) {
+ Error("%s", err.c_str());
+ return 1;
+ }
+
+ for (Node* node : nodes) {
+ node->MarkInputsChecked();
+ // Call ToolInputsProcessNode on this node's inputs, and not on itself,
+ // so that this node is not included in the output.
+ if (Edge* edge = node->in_edge(); edge != nullptr) {
+ for (Node* input : edge->inputs_) {
+ ToolInputsProcessNode(input, &deps_log_, leaf_only, include_deps);
+ }
+ if (include_deps && edge->outputs_.size() > 0) {
+ ToolInputsProcessNodeDeps(edge->outputs_[0], &deps_log_, leaf_only,
+ include_deps);
+ }
+ }
+ }
+ return 0;
+}
+
+
int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) {
if (argc == 0) {
Error("expected a target to query");
@@ -372,15 +548,33 @@
label = "|| ";
printf(" %s%s\n", label, edge->inputs_[in]->path().c_str());
}
+ if (!edge->validations_.empty()) {
+ printf(" validations:\n");
+ for (Node* validation : edge->validations_) {
+ printf(" %s\n", validation->path().c_str());
+ }
+ }
}
printf(" outputs:\n");
- for (vector<Edge*>::const_iterator edge = node->out_edges().begin();
- edge != node->out_edges().end(); ++edge) {
+ const std::vector<Edge*> out_edges = node->GetOutEdges();
+ for (vector<Edge*>::const_iterator edge = out_edges.begin();
+ edge != out_edges.end(); ++edge) {
for (vector<Node*>::iterator out = (*edge)->outputs_.begin();
out != (*edge)->outputs_.end(); ++out) {
printf(" %s\n", (*out)->path().c_str());
}
}
+ const std::vector<Edge*> validation_edges = node->GetValidationOutEdges();
+ if (!validation_edges.empty()) {
+ printf(" validation for:\n");
+ for (vector<Edge*>::const_iterator edge = validation_edges.begin();
+ edge != validation_edges.end(); ++edge) {
+ for (vector<Node*>::iterator out = (*edge)->outputs_.begin();
+ out != (*edge)->outputs_.end(); ++out) {
+ printf(" %s\n", (*out)->path().c_str());
+ }
+ }
+ }
}
return 0;
}
@@ -555,7 +749,7 @@
}
enum PrintCommandMode { PCM_Single, PCM_All };
-void PrintCommands(Edge* edge, set<Edge*>* seen, PrintCommandMode mode) {
+void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) {
if (!edge)
return;
if (!seen->insert(edge).second)
@@ -606,7 +800,7 @@
return 1;
}
- set<Edge*> seen;
+ EdgeSet seen;
for (vector<Node*>::iterator in = nodes.begin(); in != nodes.end(); ++in)
PrintCommands((*in)->in_edge(), &seen, mode);
@@ -826,6 +1020,10 @@
Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps },
{ "graph", "output graphviz dot file for targets",
Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph },
+ { "inputs", "show all (recursive) inputs for a target",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolInputs },
+ { "path", "find dependency path between two targets",
+ Tool::RUN_AFTER_LOGS, &NinjaMain::ToolPath },
{ "query", "show inputs/outputs for a path",
Tool::RUN_AFTER_LOGS, &NinjaMain::ToolQuery },
{ "targets", "list targets by their rule or depth in the DAG",
@@ -878,6 +1076,7 @@
#ifdef _WIN32
" nostatcache don't batch stat() calls per directory and cache them\n"
#endif
+" nothreads don't use threads to parallelize ninja\n"
"multiple modes can be enabled via -d FOO -d BAR\n");
return false;
} else if (name == "stats") {
@@ -895,11 +1094,14 @@
} else if (name == "nostatcache") {
g_experimental_statcache = false;
return true;
+ } else if (name == "nothreads") {
+ g_use_threads = false;
+ return true;
} else {
const char* suggestion =
SpellcheckString(name.c_str(),
"stats", "explain", "keepdepfile", "keeprsp",
- "nostatcache", NULL);
+ "nostatcache", "nothreads", NULL);
if (suggestion) {
Error("unknown debug setting '%s', did you mean '%s'?",
name.c_str(), suggestion);
@@ -912,13 +1114,18 @@
/// Set a warning flag. Returns false if Ninja should exit instead of
/// continuing.
-bool WarningEnable(const string& name, Options* options) {
+bool WarningEnable(const string& name, Options* options, BuildConfig* config) {
if (name == "list") {
printf("warning flags:\n"
" dupbuild={err,warn} multiple build lines for one target\n"
" phonycycle={err,warn} phony build statement references itself\n"
" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n"
- );
+" missingdepfile={err,warn} how to treat missing depfiles\n"
+"\n"
+" requires -o usesphonyoutputs=yes\n"
+" outputdir={err,warn} how to treat outputs that are directories\n"
+" missingoutfile={err,warn} how to treat missing output files\n"
+" oldoutput={err,warn} how to treat output files older than their inputs\n");
return false;
} else if (name == "dupbuild=err") {
options->dupe_edges_should_err = true;
@@ -938,10 +1145,38 @@
} else if (name == "depfilemulti=warn") {
options->depfile_distinct_target_lines_should_err = false;
return true;
+ } else if (name == "missingdepfile=err") {
+ config->missing_depfile_should_err = true;
+ return true;
+ } else if (name == "missingdepfile=warn") {
+ config->missing_depfile_should_err = false;
+ return true;
+ } else if (name == "outputdir=err") {
+ config->output_directory_should_err = true;
+ return true;
+ } else if (name == "outputdir=warn") {
+ config->output_directory_should_err = false;
+ return true;
+ } else if (name == "missingoutfile=err") {
+ config->missing_output_file_should_err = true;
+ return true;
+ } else if (name == "missingoutfile=warn") {
+ config->missing_output_file_should_err = false;
+ return true;
+ } else if (name == "oldoutput=err") {
+ config->old_output_should_err = true;
+ return true;
+ } else if (name == "oldoutput=warn") {
+ config->old_output_should_err = false;
+ return true;
} else {
const char* suggestion =
SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn",
- "phonycycle=err", "phonycycle=warn", NULL);
+ "phonycycle=err", "phonycycle=warn",
+ "missingdepfile=err", "missingdepfile=warn",
+ "outputdir=err", "outputdir=warn",
+ "missingoutfile=err", "missingoutfile=warn",
+ "oldoutput=err", "oldoutput=warn", NULL);
if (suggestion) {
Error("unknown warning flag '%s', did you mean '%s'?",
name.c_str(), suggestion);
@@ -952,6 +1187,46 @@
}
}
+/// Set an option flag. Returns false if Ninja should exit instead of
+/// continuing.
+bool OptionEnable(const string& name, Options* options, BuildConfig* config) {
+ if (name == "list") {
+ printf("option flags:\n"
+" usesphonyoutputs={yes,no} whether the generate uses 'phony_output's so \n"
+" that these warnings work:\n"
+" outputdir\n"
+" missingoutfile\n"
+" oldoutput\n"
+" preremoveoutputs={yes,no} whether to remove outputs before running rule\n");
+ return false;
+ } else if (name == "usesphonyoutputs=yes") {
+ config->uses_phony_outputs = true;
+ return true;
+ } else if (name == "usesphonyoutputs=no") {
+ config->uses_phony_outputs = false;
+ return true;
+ } else if (name == "preremoveoutputs=yes") {
+ config->pre_remove_output_files = true;
+ return true;
+ } else if (name == "preremoveoutputs=no") {
+ config->pre_remove_output_files = false;
+ return true;
+ } else {
+ const char* suggestion =
+ SpellcheckString(name.c_str(),
+ "usesphonyoutputs=yes", "usesphonyoutputs=no",
+ "preremoveoutputs=yes", "preremoveoutputs=no",
+ NULL);
+ if (suggestion) {
+ Error("unknown option flag '%s', did you mean '%s'?",
+ name.c_str(), suggestion);
+ } else {
+ Error("unknown option flag '%s'", name.c_str());
+ }
+ return false;
+ }
+}
+
bool NinjaMain::OpenBuildLog(bool recompact_only) {
string log_path = ".ninja_log";
if (!build_dir_.empty())
@@ -1004,14 +1279,14 @@
}
if (recompact_only) {
- bool success = deps_log_.Recompact(path, &err);
+ bool success = deps_log_.Recompact(path, disk_interface_, &err);
if (!success)
Error("failed recompaction: %s", err.c_str());
return success;
}
if (!config_.dry_run) {
- if (!deps_log_.OpenForWrite(path, &err)) {
+ if (!deps_log_.OpenForWrite(path, disk_interface_, &err)) {
Error("opening deps log: %s", err.c_str());
return false;
}
@@ -1020,18 +1295,20 @@
return true;
}
-void NinjaMain::DumpMetrics() {
- g_metrics->Report();
+void NinjaMain::DumpMetrics(Status *status) {
+ g_metrics->Report(status);
- printf("\n");
+ status->Debug("");
+
int count = (int)state_.paths_.size();
int buckets = (int)state_.paths_.bucket_count();
- printf("path->node hash load %.2f (%d entries / %d buckets)\n",
- count / (double) buckets, count, buckets);
+ status->Debug("path->node hash load %.2f (%d entries / %d buckets), %zu edges",
+ count / (double) buckets, count, buckets, state_.edges_.size());
+ DumpMemoryUsage(status);
}
bool NinjaMain::EnsureBuildDirExists() {
- build_dir_ = state_.bindings_.LookupVariable("builddir");
+ build_dir_ = state_.root_scope_.LookupVariable("builddir");
if (!build_dir_.empty() && !config_.dry_run) {
if (!disk_interface_.MakeDirs(build_dir_ + "/.") && errno != EEXIST) {
Error("creating build directory %s: %s",
@@ -1042,39 +1319,33 @@
return true;
}
-int NinjaMain::RunBuild(int argc, char** argv) {
+int NinjaMain::RunBuild(int argc, char** argv, Status* status) {
string err;
vector<Node*> targets;
if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) {
- Error("%s", err.c_str());
+ status->Error("%s", err.c_str());
return 1;
}
disk_interface_.AllowStatCache(g_experimental_statcache);
- Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
- for (size_t i = 0; i < targets.size(); ++i) {
- if (!builder.AddTarget(targets[i], &err)) {
- if (!err.empty()) {
- Error("%s", err.c_str());
- return 1;
- } else {
- // Added a target that is already up-to-date; not really
- // an error.
- }
- }
+ Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_,
+ status, start_time_millis_);
+ if (!builder.AddTargets(targets, &err)) {
+ status->Error("%s", err.c_str());
+ return 1;
}
// Make sure restat rules do not see stale timestamps.
disk_interface_.AllowStatCache(false);
if (builder.AlreadyUpToDate()) {
- printf("ninja: no work to do.\n");
+ status->Info("no work to do.");
return 0;
}
if (!builder.Build(&err)) {
- printf("ninja: build stopped: %s.\n", err.c_str());
+ status->Info("build stopped: %s.", err.c_str());
if (err.find("interrupted by user") != string::npos) {
return 2;
}
@@ -1113,8 +1384,16 @@
Options* options, BuildConfig* config) {
config->parallelism = GuessParallelism();
- enum { OPT_VERSION = 1 };
+ enum {
+ OPT_VERSION = 1,
+ OPT_FRONTEND = 2,
+ OPT_FRONTEND_FILE = 3,
+ };
const option kLongOptions[] = {
+#ifndef _WIN32
+ { "frontend", required_argument, NULL, OPT_FRONTEND },
+ { "frontend_file", required_argument, NULL, OPT_FRONTEND_FILE },
+#endif
{ "help", no_argument, NULL, 'h' },
{ "version", no_argument, NULL, OPT_VERSION },
{ "verbose", no_argument, NULL, 'v' },
@@ -1123,7 +1402,7 @@
int opt;
while (!options->tool &&
- (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions,
+ (opt = getopt_long(*argc, *argv, "d:f:j:k:l:mnt:vw:o:C:ph", kLongOptions,
NULL)) != -1) {
switch (opt) {
case 'd':
@@ -1176,15 +1455,28 @@
config->verbosity = BuildConfig::VERBOSE;
break;
case 'w':
- if (!WarningEnable(optarg, options))
+ if (!WarningEnable(optarg, options, config))
+ return 1;
+ break;
+ case 'o':
+ if (!OptionEnable(optarg, options, config))
return 1;
break;
case 'C':
options->working_dir = optarg;
break;
+ case 'p':
+ options->persistent = true;
+ break;
case OPT_VERSION:
printf("%s\n", kNinjaVersion);
return 0;
+ case OPT_FRONTEND:
+ config->frontend = optarg;
+ break;
+ case OPT_FRONTEND_FILE:
+ config->frontend_file = optarg;
+ break;
case 'h':
default:
Usage(*config);
@@ -1194,9 +1486,30 @@
*argv += optind;
*argc -= optind;
+ if (config->frontend != NULL && config->frontend_file != NULL) {
+ Fatal("only one of --frontend or --frontend_file may be specified.");
+ }
+
+ if (config->pre_remove_output_files && !config->uses_phony_outputs) {
+ Fatal("preremoveoutputs=yes requires usesphonyoutputs=yes.");
+ }
+
return -1;
}
+static void WaitForInput(const BuildConfig& config) {
+ static char* buf = nullptr;
+ static size_t len = 0;
+
+ if (config.verbosity == BuildConfig::VERBOSE) {
+ fprintf(stderr, "ninja waiting for input...\n");
+ }
+ ssize_t rc = getline(&buf, &len, stdin);
+ if (rc < 0) {
+ exit(0);
+ }
+}
+
NORETURN void real_main(int argc, char** argv) {
// Use exit() instead of return in this function to avoid potentially
// expensive cleanup when destructing NinjaMain.
@@ -1224,12 +1537,15 @@
// Don't print this if a tool is being used, so that tool output
// can be piped into a file without this string showing up.
if (!options.tool)
- printf("ninja: Entering directory `%s'\n", options.working_dir);
+ Info("Entering directory `%s'", options.working_dir);
if (chdir(options.working_dir) < 0) {
- Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno));
+ Error("chdir to '%s' - %s", options.working_dir, strerror(errno));
+ exit(1);
}
}
+ SetThreadPoolThreadCount(g_use_threads ? GetProcessorCount() : 1);
+
if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) {
// None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed
// by other tools.
@@ -1237,11 +1553,22 @@
exit((ninja.*options.tool->func)(&options, argc, argv));
}
+ Status* status = NULL;
+
// Limit number of rebuilds, to prevent infinite loops.
const int kCycleLimit = 100;
for (int cycle = 1; cycle <= kCycleLimit; ++cycle) {
NinjaMain ninja(ninja_command, config);
+ if (status == NULL) {
+#ifndef _WIN32
+ if (config.frontend != NULL || config.frontend_file != NULL)
+ status = new StatusSerializer(config);
+ else
+#endif
+ status = new StatusPrinter(config);
+ }
+
ManifestParserOptions parser_opts;
if (options.dupe_edges_should_err) {
parser_opts.dupe_edge_action_ = kDupeEdgeActionError;
@@ -1252,7 +1579,7 @@
ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts);
string err;
if (!parser.Load(options.input_file, &err)) {
- Error("%s", err.c_str());
+ status->Error("%s", err.c_str());
exit(1);
}
@@ -1269,7 +1596,7 @@
exit((ninja.*options.tool->func)(&options, argc, argv));
// Attempt to rebuild the manifest before building anything else
- if (ninja.RebuildManifest(options.input_file, &err)) {
+ if (ninja.RebuildManifest(options.input_file, &err, status)) {
// In dry_run mode the regeneration will succeed without changing the
// manifest forever. Better to return immediately.
if (config.dry_run)
@@ -1277,18 +1604,37 @@
// Start the build over with the new manifest.
continue;
} else if (!err.empty()) {
- Error("rebuilding '%s': %s", options.input_file, err.c_str());
+ status->Error("rebuilding '%s': %s", options.input_file, err.c_str());
exit(1);
}
- int result = ninja.RunBuild(argc, argv);
+ int result = 0;
+ do {
+ Stopwatch stopwatch;
+
+ if (options.persistent) {
+ WaitForInput(config);
+ stopwatch.Restart();
+ }
+
+ result = ninja.RunBuild(argc, argv, status);
+ if (options.persistent) {
+ fprintf(stderr, "build %s in %0.3f seconds\n",
+ result == 0 ? "succeeded" : "failed", stopwatch.Elapsed());
+ ninja.state_.Reset();
+ }
+ } while (options.persistent);
+
if (g_metrics)
- ninja.DumpMetrics();
+ ninja.DumpMetrics(status);
+
+ delete status;
exit(result);
}
- Error("manifest '%s' still dirty after %d tries\n",
+ status->Error("manifest '%s' still dirty after %d tries",
options.input_file, kCycleLimit);
+ delete status;
exit(1);
}
diff --git a/src/ninja_test.cc b/src/ninja_test.cc
index d642c5c..066ed43 100644
--- a/src/ninja_test.cc
+++ b/src/ninja_test.cc
@@ -25,8 +25,9 @@
#include <getopt.h>
#endif
-#include "test.h"
#include "line_printer.h"
+#include "test.h"
+#include "thread_pool.h"
struct RegisteredTest {
testing::Test* (*factory)();
@@ -104,6 +105,9 @@
*test_filter = optarg;
break;
} // else fall through.
+#if defined(__has_cpp_attribute) && __has_cpp_attribute(clang::fallthrough)
+ [[clang::fallthrough]];
+#endif
default:
Usage();
return false;
@@ -129,6 +133,8 @@
int main(int argc, char **argv) {
int tests_started = 0;
+ SetThreadPoolThreadCount(16);
+
const char* test_filter = "*";
if (!ReadFlags(&argc, &argv, &test_filter))
return 1;
diff --git a/src/parallel_map.h b/src/parallel_map.h
new file mode 100644
index 0000000..00799d8
--- /dev/null
+++ b/src/parallel_map.h
@@ -0,0 +1,109 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_PARALLEL_MAP_H_
+#define NINJA_PARALLEL_MAP_H_
+
+#include <functional>
+#include <mutex>
+#include <utility>
+#include <vector>
+
+#include "thread_pool.h"
+#include "util.h"
+
+namespace detail {
+
+template <typename T>
+struct MapFnResults {
+ MapFnResults(size_t size) : size_(size), results_(new T[size] {}) {}
+
+ using ReturnType = std::vector<T>;
+
+ template <typename Vector, typename MapFn>
+ void MapItem(Vector& vec, const MapFn& map_fn, size_t idx) {
+ assert(idx < size_);
+ results_.get()[idx] = map_fn(vec[idx]);
+ }
+
+ ReturnType Finish() {
+ ReturnType result;
+ result.reserve(size_);
+ std::move(results_.get(), results_.get() + size_,
+ std::back_inserter(result));
+ return result;
+ }
+
+private:
+ size_t size_;
+
+ /// We'd like to store the intermediate results using std::vector<T>, but
+ /// the std::vector<bool> specialization doesn't allow concurrent modification
+ /// of different elements, so use a simple array instead.
+ std::unique_ptr<T[]> results_;
+};
+
+template <>
+struct MapFnResults<void> {
+ MapFnResults(size_t) {}
+
+ using ReturnType = void;
+
+ template <typename Vector, typename MapFn>
+ void MapItem(Vector& vec, const MapFn& map_fn, size_t idx) {
+ map_fn(vec[idx]);
+ }
+
+ ReturnType Finish() {}
+};
+
+} // namespace detail
+
+template <typename T>
+std::vector<std::pair<T, T>> SplitByThreads(T total) {
+ return SplitByCount(total, GetOptimalThreadPoolJobCount());
+}
+
+/// Run |map_fn| on each element of |vec| and return a vector containing the
+/// result, in the same order as the input vector. Returns void instead if the
+/// map function returns void.
+///
+/// The map function is invoked from multiple threads, so the functor is marked
+/// const. (e.g. A "mutable" lambda isn't allowed.)
+template <typename Vector, typename MapFn>
+auto ParallelMap(ThreadPool* thread_pool, Vector&& vec, const MapFn& map_fn) ->
+ typename detail::MapFnResults<
+ typename std::remove_reference<
+ decltype(map_fn(vec[0]))>::type>::ReturnType {
+ // Identify the return type of the map function, and if it isn't void, prepare
+ // a vector to hold the result of mapping each item in the sequence.
+ using MapReturn = typename std::remove_reference<decltype(map_fn(vec[0]))>::type;
+ detail::MapFnResults<MapReturn> results(vec.size());
+
+ // Split the sequence up into groups.
+ std::vector<std::pair<size_t, size_t>> ranges = SplitByThreads(vec.size());
+ std::vector<std::function<void()>> tasks;
+ for (auto& range : ranges) {
+ tasks.emplace_back([&results, &vec, &map_fn, range] {
+ for (size_t idx = range.first; idx < range.second; ++idx) {
+ results.MapItem(vec, map_fn, idx);
+ }
+ });
+ }
+ thread_pool->RunTasks(std::move(tasks));
+
+ return results.Finish();
+}
+
+#endif // NINJA_PARALLEL_MAP_H_
diff --git a/src/proto.cc b/src/proto.cc
new file mode 100644
index 0000000..c29e621
--- /dev/null
+++ b/src/proto.cc
@@ -0,0 +1,154 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define __STDC_LIMIT_MACROS
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "proto.h"
+
+// This file and proto.h implement a minimal write-only protobuf runtime to be
+// used with headers generated by misc/generate_proto_header.py.
+
+// From https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/wire_format_lite.h
+enum WireType {
+ WIRETYPE_VARINT = 0,
+ WIRETYPE_FIXED64 = 1,
+ WIRETYPE_LENGTH_DELIMITED = 2,
+ WIRETYPE_START_GROUP = 3,
+ WIRETYPE_END_GROUP = 4,
+ WIRETYPE_FIXED32 = 5,
+};
+
+static uint8_t MakeTag(int number, WireType wire_type) {
+ const size_t FIELD_NUMBER_SHIFT = 3;
+ return (number << FIELD_NUMBER_SHIFT) | static_cast<uint8_t>(wire_type);
+}
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/io/coded_stream.h
+// Compile-time equivalent of VarintSize64().
+template <uint64_t Value>
+struct StaticVarintSize {
+ static const int value =
+ (Value < (1ULL << 7))
+ ? 1
+ : (Value < (1ULL << 14))
+ ? 2
+ : (Value < (1ULL << 21))
+ ? 3
+ : (Value < (1ULL << 28))
+ ? 4
+ : (Value < (1ULL << 35))
+ ? 5
+ : (Value < (1ULL << 42))
+ ? 6
+ : (Value < (1ULL << 49))
+ ? 7
+ : (Value < (1ULL << 56))
+ ? 8
+ : (Value < (1ULL << 63))
+ ? 9
+ : 10;
+};
+
+static void WriteVarint32WithType(std::ostream* output, int number,
+ uint32_t value, WireType type) {
+ uint8_t buf[StaticVarintSize<UINT32_MAX>::value + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, type);
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteVarint32NoTag(std::ostream* output, uint32_t value) {
+ uint8_t buf[StaticVarintSize<UINT32_MAX>::value];
+ uint8_t *target = buf;
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteVarint32(std::ostream* output, int number, uint32_t value) {
+ WriteVarint32WithType(output, number, value, WIRETYPE_VARINT);
+}
+
+void WriteVarint32SignExtended(std::ostream* output, int number,
+ int32_t value) {
+ if (value < 0) {
+ WriteVarint64(output, number, static_cast<uint64_t>(value));
+ } else {
+ WriteVarint32(output, number, static_cast<uint32_t>(value));
+ }
+}
+
+void WriteVarint64(std::ostream* output, int number, uint64_t value) {
+ uint8_t buf[StaticVarintSize<UINT64_MAX>::value + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_VARINT);
+
+ while (value >= 0x80) {
+ *target = static_cast<uint8_t>(value | 0x80);
+ value >>= 7;
+ ++target;
+ }
+ *target++ = static_cast<uint8_t>(value);
+
+ output->write(reinterpret_cast<char*>(buf), target - buf);
+}
+
+void WriteFixed32(std::ostream* output, int number, uint32_t value) {
+ uint8_t buf[sizeof(value) + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_FIXED32);
+ memcpy(target, &value, sizeof(value));
+
+ output->write(reinterpret_cast<char*>(buf), sizeof(buf));
+}
+
+void WriteFixed64(std::ostream* output, int number, uint64_t value) {
+ uint8_t buf[sizeof(value) + 1];
+ uint8_t *target = buf;
+
+ *target++ = MakeTag(number, WIRETYPE_FIXED64);
+ memcpy(target, &value, sizeof(value));
+
+ output->write(reinterpret_cast<char*>(buf), sizeof(buf));
+}
+
+void WriteString(std::ostream* output, int number, const std::string& value) {
+ WriteLengthDelimited(output, number, value.size());
+ output->write(value.data(), value.size());
+}
+
+void WriteLengthDelimited(std::ostream* output, int number, size_t size) {
+ WriteVarint32WithType(output, number, size, WIRETYPE_LENGTH_DELIMITED);
+}
diff --git a/src/proto.h b/src/proto.h
new file mode 100644
index 0000000..8552719
--- /dev/null
+++ b/src/proto.h
@@ -0,0 +1,110 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_PROTO_H_
+#define NINJA_PROTO_H_
+
+#include <iostream>
+
+// This file and proto.cc implement a minimal write-only protobuf runtime to be
+// used with headers generated by misc/generate_proto_header.py.
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/wire_format_lite.h
+static inline uint32_t ZigZagEncode32(int32_t n) {
+ // Note: the right-shift must be arithmetic
+ return (static_cast<uint32_t>(n) << 1) ^ (n >> 31);
+}
+
+static inline uint64_t ZigZagEncode64(int64_t n) {
+ // Note: the right-shift must be arithmetic
+ return (static_cast<uint64_t>(n) << 1) ^ (n >> 63);
+}
+
+// Based on https://github.com/google/protobuf/blob/3.0.x/src/google/protobuf/io/coded_stream.h
+static inline size_t VarintSize32(uint32_t value) {
+ if (value < (1 << 7)) {
+ return 1;
+ } else if (value < (1 << 14)) {
+ return 2;
+ } else if (value < (1 << 21)) {
+ return 3;
+ } else if (value < (1 << 28)) {
+ return 4;
+ } else {
+ return 5;
+ }
+}
+
+static inline size_t VarintSize32SignExtended(int32_t value) {
+ if (value < 0) {
+ return 10; // TODO(kenton): Make this a symbolic constant.
+ } else {
+ return VarintSize32(static_cast<uint32_t>(value));
+ }
+}
+
+static inline size_t VarintSize64(uint64_t value) {
+ if (value < (1ull << 35)) {
+ if (value < (1ull << 7)) {
+ return 1;
+ } else if (value < (1ull << 14)) {
+ return 2;
+ } else if (value < (1ull << 21)) {
+ return 3;
+ } else if (value < (1ull << 28)) {
+ return 4;
+ } else {
+ return 5;
+ }
+ } else {
+ if (value < (1ull << 42)) {
+ return 6;
+ } else if (value < (1ull << 49)) {
+ return 7;
+ } else if (value < (1ull << 56)) {
+ return 8;
+ } else if (value < (1ull << 63)) {
+ return 9;
+ } else {
+ return 10;
+ }
+ }
+}
+
+static inline size_t VarintSizeBool(bool /*value*/) {
+ return 1;
+}
+
+static inline size_t FixedSize32(uint32_t /*value*/) {
+ return 4;
+}
+
+static inline size_t FixedSize64(uint64_t /*value*/) {
+ return 8;
+}
+
+static inline size_t StringSize(const std::string& value) {
+ return VarintSize32(value.size()) + value.size();
+}
+
+void WriteVarint32(std::ostream* output, int number, uint32_t value);
+void WriteVarint32NoTag(std::ostream* output, uint32_t value);
+void WriteVarint32SignExtended(std::ostream* output, int number, int32_t value);
+void WriteVarint64(std::ostream* output, int number, uint64_t value);
+void WriteFixed32(std::ostream* output, int number, uint32_t value);
+void WriteFixed64(std::ostream* output, int number, uint64_t value);
+void WriteString(std::ostream* output, int number, const std::string& value);
+void WriteLengthDelimited(std::ostream* output, int number, size_t size);
+
+#endif
diff --git a/src/state.cc b/src/state.cc
index 9b3c7e1..f8e47aa 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -22,6 +22,9 @@
#include "metrics.h"
#include "util.h"
+Pool::Pool(const HashedStrView& name, int depth) : name_(name), depth_(depth) {
+ pos_.base = new BasePosition {{ &State::kBuiltinScope, 0 }}; // leaked
+}
void Pool::EdgeScheduled(const Edge& edge) {
if (depth_ != 0)
@@ -38,7 +41,7 @@
delayed_.insert(edge);
}
-void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) {
+void Pool::RetrieveReadyEdges(EdgeSet* ready_queue) {
DelayedEdges::iterator it = delayed_.begin();
while (it != delayed_.end()) {
Edge* edge = *it;
@@ -61,63 +64,77 @@
}
}
-// static
-bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) {
- if (!a) return b;
- if (!b) return false;
- int weight_diff = a->weight() - b->weight();
- return ((weight_diff < 0) || (weight_diff == 0 && a < b));
-}
-
+Scope State::kBuiltinScope({});
Pool State::kDefaultPool("", 0);
Pool State::kConsolePool("console", 1);
-const Rule State::kPhonyRule("phony");
+Rule State::kPhonyRule("phony");
State::State() {
- bindings_.AddRule(&kPhonyRule);
+ // Reserve scope position (root, 0) for built-in rules.
+ root_scope_.AllocDecls(1);
+
+ AddBuiltinRule(&kPhonyRule);
AddPool(&kDefaultPool);
AddPool(&kConsolePool);
}
-void State::AddPool(Pool* pool) {
- assert(LookupPool(pool->name()) == NULL);
- pools_[pool->name()] = pool;
+// Add a built-in rule at the top of the root scope.
+void State::AddBuiltinRule(Rule* rule) {
+ root_scope_.AddRule(rule);
}
-Pool* State::LookupPool(const string& pool_name) {
- map<string, Pool*>::iterator i = pools_.find(pool_name);
- if (i == pools_.end())
- return NULL;
- return i->second;
+bool State::AddPool(Pool* pool) {
+ return pools_.insert({ pool->name_hashed(), pool }).second;
}
Edge* State::AddEdge(const Rule* rule) {
Edge* edge = new Edge();
+ edge->pos_.base = new BasePosition {{ &root_scope_, 0 }}; // leaked
edge->rule_ = rule;
edge->pool_ = &State::kDefaultPool;
- edge->env_ = &bindings_;
+ edge->id_ = edges_.size();
edges_.push_back(edge);
return edge;
}
-Node* State::GetNode(StringPiece path, uint64_t slash_bits) {
- Node* node = LookupNode(path);
- if (node)
- return node;
- node = new Node(path.AsString(), slash_bits);
- paths_[node->path()] = node;
- return node;
+Pool* State::LookupPool(const HashedStrView& pool_name) {
+ auto i = pools_.find(pool_name);
+ if (i == pools_.end())
+ return nullptr;
+ return i->second;
}
-Node* State::LookupNode(StringPiece path) const {
- METRIC_RECORD("lookup node");
- Paths::const_iterator i = paths_.find(path);
- if (i != paths_.end())
- return i->second;
- return NULL;
+Pool* State::LookupPoolAtPos(const HashedStrView& pool_name,
+ DeclIndex dfs_location) {
+ Pool* result = LookupPool(pool_name);
+ if (result == nullptr) return nullptr;
+ return result->dfs_location() < dfs_location ? result : nullptr;
}
-Node* State::SpellcheckNode(const string& path) {
+Node* State::GetNode(const HashedStrView& path, uint64_t slash_bits) {
+ if (Node** opt_node = paths_.Lookup(path))
+ return *opt_node;
+ // Create a new node and try to insert it.
+ std::unique_ptr<Node> node(new Node(path, slash_bits));
+ if (paths_.insert({node->path_hashed(), node.get()}).second)
+ return node.release();
+ // Another thread beat us to it. Use its node instead.
+ return *paths_.Lookup(path);
+}
+
+Node* State::LookupNode(const HashedStrView& path) const {
+ if (Node* const* opt_node = paths_.Lookup(path))
+ return *opt_node;
+ return nullptr;
+}
+
+Node* State::LookupNodeAtPos(const HashedStrView& path,
+ DeclIndex dfs_location) const {
+ Node* result = LookupNode(path);
+ return result && result->dfs_location() < dfs_location ? result : nullptr;
+}
+
+Node* State::SpellcheckNode(StringPiece path) {
const bool kAllowReplacements = true;
const int kMaxValidEditDistance = 3;
@@ -125,7 +142,7 @@
Node* result = NULL;
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i) {
int distance = EditDistance(
- i->first, path, kAllowReplacements, kMaxValidEditDistance);
+ i->first.str_view(), path, kAllowReplacements, kMaxValidEditDistance);
if (distance < min_distance && i->second) {
min_distance = distance;
result = i->second;
@@ -149,16 +166,6 @@
return true;
}
-bool State::AddDefault(StringPiece path, string* err) {
- Node* node = LookupNode(path);
- if (!node) {
- *err = "unknown target '" + path.AsString() + "'";
- return false;
- }
- defaults_.push_back(node);
- return true;
-}
-
vector<Node*> State::RootNodes(string* err) const {
vector<Node*> root_nodes;
// Search for nodes with no output.
@@ -166,7 +173,7 @@
e != edges_.end(); ++e) {
for (vector<Node*>::const_iterator out = (*e)->outputs_.begin();
out != (*e)->outputs_.end(); ++out) {
- if ((*out)->out_edges().empty())
+ if (!(*out)->has_out_edge())
root_nodes.push_back(*out);
}
}
@@ -181,12 +188,19 @@
return defaults_.empty() ? RootNodes(err) : defaults_;
}
+DeclIndex State::AllocDfsLocation(DeclIndex count) {
+ DeclIndex result = dfs_location_;
+ dfs_location_ += count;
+ return result;
+}
+
void State::Reset() {
for (Paths::iterator i = paths_.begin(); i != paths_.end(); ++i)
i->second->ResetState();
for (vector<Edge*>::iterator e = edges_.begin(); e != edges_.end(); ++e) {
(*e)->outputs_ready_ = false;
(*e)->mark_ = Edge::VisitNone;
+ (*e)->precomputed_dirtiness_ = false;
}
}
@@ -201,9 +215,7 @@
}
if (!pools_.empty()) {
printf("resource_pools:\n");
- for (map<string, Pool*>::const_iterator it = pools_.begin();
- it != pools_.end(); ++it)
- {
+ for (auto it = pools_.begin(); it != pools_.end(); ++it) {
if (!it->second->name().empty()) {
it->second->Dump();
}
diff --git a/src/state.h b/src/state.h
index 6fe886c..db18d60 100644
--- a/src/state.h
+++ b/src/state.h
@@ -16,13 +16,16 @@
#define NINJA_STATE_H_
#include <map>
+#include <memory>
#include <set>
#include <string>
#include <vector>
using namespace std;
#include "eval_env.h"
-#include "hash_map.h"
+#include "graph.h"
+#include "string_piece.h"
+#include "concurrent_hash_map.h"
#include "util.h"
struct Edge;
@@ -38,13 +41,17 @@
/// the total scheduled weight diminishes enough (i.e. when a scheduled edge
/// completes).
struct Pool {
- Pool(const string& name, int depth)
- : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) {}
+ Pool() {}
+
